diff --git a/ciao-controller/compute_test.go b/ciao-controller/compute_test.go index 685957ab7..b7e873859 100644 --- a/ciao-controller/compute_test.go +++ b/ciao-controller/compute_test.go @@ -233,12 +233,7 @@ func TestDeleteServer(t *testing.T) { time.Sleep(2 * time.Second) - c := client.AddCmdChan(ssntp.STATS) - go client.SendStatsCmd() - _, err = client.GetCmdChanResult(c, ssntp.STATS) - if err != nil { - t.Fatal(err) - } + sendStatsCmd(client, t) time.Sleep(2 * time.Second) @@ -280,12 +275,7 @@ func TestServersActionStart(t *testing.T) { time.Sleep(2 * time.Second) - c := client.AddCmdChan(ssntp.STATS) - go client.SendStatsCmd() - _, err = client.GetCmdChanResult(c, ssntp.STATS) - if err != nil { - t.Fatal(err) - } + sendStatsCmd(client, t) time.Sleep(1 * time.Second) @@ -296,12 +286,7 @@ func TestServersActionStart(t *testing.T) { time.Sleep(1 * time.Second) - c = client.AddCmdChan(ssntp.STATS) - go client.SendStatsCmd() - _, err = client.GetCmdChanResult(c, ssntp.STATS) - if err != nil { - t.Fatal(err) - } + sendStatsCmd(client, t) time.Sleep(1 * time.Second) @@ -342,12 +327,7 @@ func TestServersActionStop(t *testing.T) { time.Sleep(2 * time.Second) - c := client.AddCmdChan(ssntp.STATS) - go client.SendStatsCmd() - _, err = client.GetCmdChanResult(c, ssntp.STATS) - if err != nil { - t.Fatal(err) - } + sendStatsCmd(client, t) time.Sleep(1 * time.Second) @@ -388,12 +368,7 @@ func TestServerActionStop(t *testing.T) { time.Sleep(2 * time.Second) - c := client.AddCmdChan(ssntp.STATS) - go client.SendStatsCmd() - _, err = client.GetCmdChanResult(c, ssntp.STATS) - if err != nil { - t.Fatal(err) - } + sendStatsCmd(client, t) time.Sleep(1 * time.Second) @@ -422,35 +397,25 @@ func TestServerActionStart(t *testing.T) { time.Sleep(1 * time.Second) - c := client.AddCmdChan(ssntp.STATS) - go client.SendStatsCmd() - _, err = client.GetCmdChanResult(c, ssntp.STATS) - if err != nil { - t.Fatal(err) - } + sendStatsCmd(client, t) time.Sleep(1 * time.Second) - c = server.AddCmdChan(ssntp.STOP) + serverCh := server.AddCmdChan(ssntp.STOP) err = context.stopInstance(servers.Servers[0].ID) if err != nil { t.Fatal(err) } - _, err = server.GetCmdChanResult(c, ssntp.STOP) + _, err = server.GetCmdChanResult(serverCh, ssntp.STOP) if err != nil { t.Fatal(err) } time.Sleep(1 * time.Second) - c = client.AddCmdChan(ssntp.STATS) - go client.SendStatsCmd() - _, err = client.GetCmdChanResult(c, ssntp.STATS) - if err != nil { - t.Fatal(err) - } + sendStatsCmd(client, t) time.Sleep(1 * time.Second) @@ -924,7 +889,7 @@ func TestListTraces(t *testing.T) { client := testStartTracedWorkload(t) defer client.Ssntp.Close() - client.SendTrace() + sendTraceReportEvent(client, t) time.Sleep(2 * time.Second) @@ -1010,7 +975,7 @@ func TestTraceData(t *testing.T) { client := testStartTracedWorkload(t) defer client.Ssntp.Close() - client.SendTrace() + sendTraceReportEvent(client, t) time.Sleep(2 * time.Second) diff --git a/ciao-controller/controller_test.go b/ciao-controller/controller_test.go index 46b13cc7a..85bbd9576 100644 --- a/ciao-controller/controller_test.go +++ b/ciao-controller/controller_test.go @@ -21,6 +21,7 @@ import ( "fmt" "net" "os" + "sync" "testing" "time" @@ -258,6 +259,34 @@ func TestStartWorkloadLaunchCNCI(t *testing.T) { } +func sendTraceReportEvent(client *testutil.SsntpTestClient, t *testing.T) { + clientCh := client.AddEventChan(ssntp.TraceReport) + serverCh := server.AddEventChan(ssntp.TraceReport) + go client.SendTrace() + _, err := client.GetEventChanResult(clientCh, ssntp.TraceReport) + if err != nil { + t.Fatal(err) + } + _, err = server.GetEventChanResult(serverCh, ssntp.TraceReport) + if err != nil { + t.Fatal(err) + } +} + +func sendStatsCmd(client *testutil.SsntpTestClient, t *testing.T) { + clientCh := client.AddCmdChan(ssntp.STATS) + serverCh := server.AddCmdChan(ssntp.STATS) + go client.SendStatsCmd() + _, err := client.GetCmdChanResult(clientCh, ssntp.STATS) + if err != nil { + t.Fatal(err) + } + _, err = server.GetCmdChanResult(serverCh, ssntp.STATS) + if err != nil { + t.Fatal(err) + } +} + // TBD: for the launch CNCI tests, I really need to create a fake // network node and test that way. @@ -267,11 +296,9 @@ func TestDeleteInstance(t *testing.T) { client, instances := testStartWorkload(t, 1, false, reason) defer client.Ssntp.Close() - time.Sleep(1 * time.Second) - - client.SendStatsCmd() + sendStatsCmd(client, t) - c := server.AddCmdChan(ssntp.DELETE) + serverCh := server.AddCmdChan(ssntp.DELETE) time.Sleep(1 * time.Second) @@ -280,7 +307,7 @@ func TestDeleteInstance(t *testing.T) { t.Fatal(err) } - result, err := server.GetCmdChanResult(c, ssntp.DELETE) + result, err := server.GetCmdChanResult(serverCh, ssntp.DELETE) if err != nil { t.Fatal(err) } @@ -295,11 +322,9 @@ func TestStopInstance(t *testing.T) { client, instances := testStartWorkload(t, 1, false, reason) defer client.Ssntp.Close() - time.Sleep(1 * time.Second) - - client.SendStatsCmd() + sendStatsCmd(client, t) - c := server.AddCmdChan(ssntp.STOP) + serverCh := server.AddCmdChan(ssntp.STOP) time.Sleep(1 * time.Second) @@ -308,7 +333,7 @@ func TestStopInstance(t *testing.T) { t.Fatal(err) } - result, err := server.GetCmdChanResult(c, ssntp.STOP) + result, err := server.GetCmdChanResult(serverCh, ssntp.STOP) if err != nil { t.Fatal(err) } @@ -325,9 +350,10 @@ func TestRestartInstance(t *testing.T) { time.Sleep(1 * time.Second) - client.SendStatsCmd() + sendStatsCmd(client, t) - c := server.AddCmdChan(ssntp.STOP) + serverCh := server.AddCmdChan(ssntp.STOP) + clientCh := client.AddCmdChan(ssntp.STOP) time.Sleep(1 * time.Second) @@ -336,7 +362,11 @@ func TestRestartInstance(t *testing.T) { t.Fatal(err) } - result, err := server.GetCmdChanResult(c, ssntp.STOP) + result, err := server.GetCmdChanResult(serverCh, ssntp.STOP) + if err != nil { + t.Fatal(err) + } + _, err = client.GetCmdChanResult(clientCh, ssntp.STOP) if err != nil { t.Fatal(err) } @@ -345,11 +375,10 @@ func TestRestartInstance(t *testing.T) { } // now attempt to restart - time.Sleep(1 * time.Second) - client.SendStatsCmd() + sendStatsCmd(client, t) - c = server.AddCmdChan(ssntp.RESTART) + serverCh = server.AddCmdChan(ssntp.RESTART) time.Sleep(1 * time.Second) @@ -358,7 +387,7 @@ func TestRestartInstance(t *testing.T) { t.Fatal(err) } - result, err = server.GetCmdChanResult(c, ssntp.RESTART) + result, err = server.GetCmdChanResult(serverCh, ssntp.RESTART) if err != nil { t.Fatal(err) } @@ -374,7 +403,7 @@ func TestEvacuateNode(t *testing.T) { } defer client.Ssntp.Close() - c := server.AddCmdChan(ssntp.EVACUATE) + serverCh := server.AddCmdChan(ssntp.EVACUATE) // ok to not send workload first? @@ -383,7 +412,7 @@ func TestEvacuateNode(t *testing.T) { t.Error(err) } - result, err := server.GetCmdChanResult(c, ssntp.EVACUATE) + result, err := server.GetCmdChanResult(serverCh, ssntp.EVACUATE) if err != nil { t.Fatal(err) } @@ -398,9 +427,9 @@ func TestInstanceDeletedEvent(t *testing.T) { client, instances := testStartWorkload(t, 1, false, reason) defer client.Ssntp.Close() - time.Sleep(1 * time.Second) + sendStatsCmd(client, t) - client.SendStatsCmd() + serverCh := server.AddCmdChan(ssntp.DELETE) time.Sleep(1 * time.Second) @@ -409,53 +438,29 @@ func TestInstanceDeletedEvent(t *testing.T) { t.Fatal(err) } - time.Sleep(1 * time.Second) - - client.SendDeleteEvent(instances[0].ID) - - time.Sleep(1 * time.Second) - - // try to get instance info - _, err = context.ds.GetInstance(instances[0].ID) - if err == nil { - t.Error("Instance not deleted") - } -} - -func TestLaunchCNCI(t *testing.T) { - netClient, err := testutil.NewSsntpTestClientConnection("LaunchCNCI", ssntp.NETAGENT, testutil.NetAgentUUID) + _, err = server.GetCmdChanResult(serverCh, ssntp.DELETE) if err != nil { t.Fatal(err) } - defer netClient.Ssntp.Close() - - c := server.AddCmdChan(ssntp.START) - - id := uuid.Generate().String() - - // this blocks till it get success or failure - go context.addTenant(id) - result, err := server.GetCmdChanResult(c, ssntp.START) + clientEvtCh := client.AddEventChan(ssntp.InstanceDeleted) + serverEvtCh := server.AddEventChan(ssntp.InstanceDeleted) + go client.SendDeleteEvent(instances[0].ID) + _, err = client.GetEventChanResult(clientEvtCh, ssntp.InstanceDeleted) if err != nil { t.Fatal(err) } - if result.TenantUUID != id { - t.Fatal("Did not get correct tenant ID") - } - if !result.CNCI { - t.Fatal("this is not a CNCI launch request") - } - - time.Sleep(2 * time.Second) - - tenant, err := context.ds.GetTenant(id) - if err != nil || tenant == nil { + _, err = server.GetEventChanResult(serverEvtCh, ssntp.InstanceDeleted) + if err != nil { t.Fatal(err) } - if tenant.CNCIIP == "" { - t.Fatal("CNCI Info not updated") + time.Sleep(1 * time.Second) + + // try to get instance info + _, err = context.ds.GetInstance(instances[0].ID) + if err == nil { + t.Error("Instance not deleted") } } @@ -480,13 +485,9 @@ func TestStopFailure(t *testing.T) { client.StopFail = true client.StopFailReason = payloads.StopNoInstance - time.Sleep(1 * time.Second) - - client.SendStatsCmd() - - time.Sleep(1 * time.Second) + sendStatsCmd(client, t) - c := server.AddCmdChan(ssntp.STOP) + serverCh := server.AddCmdChan(ssntp.STOP) time.Sleep(1 * time.Second) @@ -495,7 +496,7 @@ func TestStopFailure(t *testing.T) { t.Fatal(err) } - result, err := server.GetCmdChanResult(c, ssntp.STOP) + result, err := server.GetCmdChanResult(serverCh, ssntp.STOP) if err != nil { t.Fatal(err) } @@ -532,20 +533,23 @@ func TestRestartFailure(t *testing.T) { client.RestartFail = true client.RestartFailReason = payloads.RestartLaunchFailure - time.Sleep(1 * time.Second) - - client.SendStatsCmd() + sendStatsCmd(client, t) time.Sleep(1 * time.Second) - c := server.AddCmdChan(ssntp.STOP) + serverCh := server.AddCmdChan(ssntp.STOP) + clientCh := client.AddCmdChan(ssntp.STOP) err := context.stopInstance(instances[0].ID) if err != nil { t.Fatal(err) } - result, err := server.GetCmdChanResult(c, ssntp.STOP) + _, err = client.GetCmdChanResult(clientCh, ssntp.STOP) + if err != nil { + t.Fatal(err) + } + result, err := server.GetCmdChanResult(serverCh, ssntp.STOP) if err != nil { t.Fatal(err) } @@ -553,20 +557,18 @@ func TestRestartFailure(t *testing.T) { t.Fatal("Did not get correct Instance ID") } - time.Sleep(1 * time.Second) - - client.SendStatsCmd() + sendStatsCmd(client, t) time.Sleep(1 * time.Second) - c = server.AddCmdChan(ssntp.RESTART) + serverCh = server.AddCmdChan(ssntp.RESTART) err = context.restartInstance(instances[0].ID) if err != nil { t.Fatal(err) } - result, err = server.GetCmdChanResult(c, ssntp.RESTART) + result, err = server.GetCmdChanResult(serverCh, ssntp.RESTART) if err != nil { t.Fatal(err) } @@ -625,7 +627,8 @@ func testStartTracedWorkload(t *testing.T) *testutil.SsntpTestClient { t.Fatal("No workloads, expected len(wls) > 0, got len(wls) == 0") } - c := client.AddCmdChan(ssntp.START) + clientCh := client.AddCmdChan(ssntp.START) + serverCh := server.AddCmdChan(ssntp.START) instances, err := context.startWorkload(wls[0].ID, tenant.ID, 1, true, "testtrace1") if err != nil { @@ -635,7 +638,11 @@ func testStartTracedWorkload(t *testing.T) *testutil.SsntpTestClient { t.Fatalf("Wrong number of instances, expected 1, got %d", len(instances)) } - result, err := client.GetCmdChanResult(c, ssntp.START) + _, err = client.GetCmdChanResult(clientCh, ssntp.START) + if err != nil { + t.Fatal(err) + } + result, err := server.GetCmdChanResult(serverCh, ssntp.START) if err != nil { t.Fatal(err) } @@ -668,7 +675,8 @@ func testStartWorkload(t *testing.T, num int, fail bool, reason payloads.StartFa t.Fatal("No workloads, expected len(wls) > 0, got len(wls) == 0") } - c := client.AddCmdChan(ssntp.START) + clientCmdCh := client.AddCmdChan(ssntp.START) + clientErrCh := client.AddErrorChan(ssntp.StartFailure) client.StartFail = fail client.StartFailReason = reason @@ -680,7 +688,14 @@ func testStartWorkload(t *testing.T, num int, fail bool, reason payloads.StartFa t.Fatalf("Wrong number of instances, expected %d, got %d", len(instances), num) } - result, err := client.GetCmdChanResult(c, ssntp.START) + if fail == true { + _, err := client.GetErrorChanResult(clientErrCh, ssntp.StartFailure) + if err == nil { // unexpected success + t.Fatal(err) + } + } + + result, err := client.GetCmdChanResult(clientCmdCh, ssntp.START) if fail == true && err == nil { // unexpected success t.Fatal(err) } @@ -694,14 +709,13 @@ func testStartWorkload(t *testing.T, num int, fail bool, reason payloads.StartFa return client, instances } -// TestStartWorkloadLaunchCNCI starts a test CNCI // NOTE: the caller is responsible for calling Ssntp.Close() on the *SsntpTestClient func testStartWorkloadLaunchCNCI(t *testing.T, num int) (*testutil.SsntpTestClient, []*types.Instance) { netClient, err := testutil.NewSsntpTestClientConnection("StartWorkloadLaunchCNCI", ssntp.NETAGENT, testutil.NetAgentUUID) if err != nil { t.Fatal(err) } - // caller of TestStartWorkloadLaunchCNCI() owns doing the close + // caller of testStartWorkloadLaunchCNCI() owns doing the close //defer netClient.Ssntp.Close() wls, err := context.ds.GetWorkloads() @@ -712,14 +726,20 @@ func testStartWorkloadLaunchCNCI(t *testing.T, num int) (*testutil.SsntpTestClie t.Fatal("No workloads, expected len(wls) > 0, got len(wls) == 0") } - c := server.AddCmdChan(ssntp.START) + serverCmdCh := server.AddCmdChan(ssntp.START) + netClientCmdCh := netClient.AddCmdChan(ssntp.START) - id := uuid.Generate().String() + newTenant := uuid.Generate().String() // random ~= new tenant and thus triggers start of a CNCI + // trigger the START command flow, and await results + // NOTE: "instances" is shared with the go subroutine and we must + // insure consistency between parent and child processes var instances []*types.Instance + var wg sync.WaitGroup + wg.Add(1) go func() { - instances, err = context.startWorkload(wls[0].ID, id, 1, false, "") + instances, err = context.startWorkload(wls[0].ID, newTenant, 1, false, "") if err != nil { t.Fatal(err) } @@ -727,13 +747,19 @@ func testStartWorkloadLaunchCNCI(t *testing.T, num int) (*testutil.SsntpTestClie if len(instances) != 1 { t.Fatalf("Wrong number of instances, expected 1, got %d", len(instances)) } + wg.Done() }() - result, err := server.GetCmdChanResult(c, ssntp.START) + _, err = netClient.GetCmdChanResult(netClientCmdCh, ssntp.START) + if err != nil { + t.Fatal(err) + } + result, err := server.GetCmdChanResult(serverCmdCh, ssntp.START) if err != nil { t.Fatal(err) } - if result.TenantUUID != id { + + if result.TenantUUID != newTenant { t.Fatal("Did not get correct tenant ID") } @@ -741,14 +767,36 @@ func testStartWorkloadLaunchCNCI(t *testing.T, num int) (*testutil.SsntpTestClie t.Fatal("this is not a CNCI launch request") } - c = server.AddCmdChan(ssntp.START) + // start a test CNCI client + cnciClient, err := testutil.NewSsntpTestClientConnection("StartWorkloadLaunchCNCI", ssntp.CNCIAGENT, newTenant) + if err != nil { + t.Fatal(err) + } - result, err = server.GetCmdChanResult(c, ssntp.START) + // make CNCI send an ssntp.ConcentratorInstanceAdded event, and await results + cnciEventCh := cnciClient.AddEventChan(ssntp.ConcentratorInstanceAdded) + serverEventCh := server.AddEventChan(ssntp.ConcentratorInstanceAdded) + tenantCNCI, _ := context.ds.GetTenantCNCISummary(result.InstanceUUID) + go cnciClient.SendConcentratorAddedEvent(result.InstanceUUID, newTenant, testutil.CNCIIP, tenantCNCI[0].MACAddress) + result, err = cnciClient.GetEventChanResult(cnciEventCh, ssntp.ConcentratorInstanceAdded) if err != nil { t.Fatal(err) } - if result.InstanceUUID != instances[0].ID { - t.Fatal("Did not get correct Instance ID") + _, err = server.GetEventChanResult(serverEventCh, ssntp.ConcentratorInstanceAdded) + if err != nil { + t.Fatal(err) + } + + // shutdown the test CNCI client + cnciClient.Ssntp.Close() + + if result.InstanceUUID != tenantCNCI[0].InstanceID { + t.Fatalf("Did not get correct Instance ID, got %s, expected %s", result.InstanceUUID, tenantCNCI[0].InstanceID) + } + + wg.Wait() // for "instances" + if instances == nil { + t.Fatal("did not receive instance") } return netClient, instances @@ -763,7 +811,6 @@ func TestMain(m *testing.M) { // create fake ssntp server testutil.StartTestServer(&server) - defer server.Ssntp.Stop() context = new(controller) context.ds = new(datastore.Datastore) @@ -797,7 +844,6 @@ func TestMain(m *testing.M) { } id := testutil.StartIdentityServer(testIdentityConfig) - defer id.Close() idConfig := identityConfig{ endpoint: id.URL, @@ -820,6 +866,8 @@ func TestMain(m *testing.M) { context.client.Disconnect() context.ds.Exit() + id.Close() + server.Ssntp.Stop() os.Remove("./ciao-controller-test.db") os.Remove("./ciao-controller-test.db-shm") diff --git a/ciao-scheduler/scheduler.go b/ciao-scheduler/scheduler.go index de8637f8d..7627d0202 100644 --- a/ciao-scheduler/scheduler.go +++ b/ciao-scheduler/scheduler.go @@ -480,10 +480,6 @@ func (sched *ssntpSchedulerServer) getConcentratorUUID(event ssntp.Event, payloa var ev payloads.EventTenantRemoved err := yaml.Unmarshal(payload, &ev) return ev.TenantRemoved.ConcentratorUUID, err - case ssntp.PublicIPAssigned: - var ev payloads.EventPublicIPAssigned - err := yaml.Unmarshal(payload, &ev) - return ev.AssignedIP.ConcentratorUUID, err } } @@ -723,8 +719,6 @@ func (sched *ssntpSchedulerServer) EventForward(uuid string, event ssntp.Event, case ssntp.TenantAdded: fallthrough case ssntp.TenantRemoved: - fallthrough - case ssntp.PublicIPAssigned: dest = sched.fwdEventToCNCI(event, payload) } @@ -958,6 +952,10 @@ func setSSNTPForwardRules(sched *ssntpSchedulerServer) { Operand: ssntp.DeleteFailure, Dest: ssntp.Controller, }, + { // all PublicIPAssigned events go to all Controllers + Operand: ssntp.PublicIPAssigned, + Dest: ssntp.Controller, + }, { // all START command are processed by the Command forwarder Operand: ssntp.START, CommandForward: sched, @@ -986,10 +984,6 @@ func setSSNTPForwardRules(sched *ssntpSchedulerServer) { Operand: ssntp.TenantRemoved, EventForward: sched, }, - { // all PublicIPAssigned events are processed by the Event forwarder - Operand: ssntp.PublicIPAssigned, - EventForward: sched, - }, } } diff --git a/ciao-scheduler/scheduler_internal_test.go b/ciao-scheduler/scheduler_internal_test.go index edd6a973c..9de425ee3 100644 --- a/ciao-scheduler/scheduler_internal_test.go +++ b/ciao-scheduler/scheduler_internal_test.go @@ -579,9 +579,11 @@ func TestStartWorkload(t *testing.T) { fwd, uuid := startWorkload(sched, controllerUUID, []byte(testutil.CNCIStartYaml)) decision := fwd.Decision() recipients := fwd.Recipients() - if decision != ssntp.Forward || - uuid != testutil.CNCIUUID { - t.Errorf("unable to start CNCI, got decision=0x%x, workload uuid=%s", decision, uuid) + if decision != ssntp.Forward { + t.Errorf("bad decision, got 0x%x, expected 0x%x", decision, ssntp.Forward) + } + if uuid != testutil.CNCIInstanceUUID { + t.Errorf("bad uuid, got %s, expected %s", uuid, testutil.CNCIInstanceUUID) } for _, dest = range recipients[:] { if sched.nnMap[dest] == nil { diff --git a/ciao-scheduler/scheduler_ssntp_test.go b/ciao-scheduler/scheduler_ssntp_test.go index cd250b45e..7a708a8f7 100644 --- a/ciao-scheduler/scheduler_ssntp_test.go +++ b/ciao-scheduler/scheduler_ssntp_test.go @@ -37,6 +37,7 @@ var server *ssntpSchedulerServer var controller *testutil.SsntpTestController var agent *testutil.SsntpTestClient var netAgent *testutil.SsntpTestClient +var cnciAgent *testutil.SsntpTestClient // these status sends need to come early so the agents are marked online // for later ssntp.START's @@ -95,6 +96,32 @@ func TestSendNetAgentStatus(t *testing.T) { server.nnMutex.Unlock() } +func TestCNCIStart(t *testing.T) { + netAgentCh := netAgent.AddCmdChan(ssntp.START) + + go controller.Ssntp.SendCommand(ssntp.START, []byte(testutil.CNCIStartYaml)) + + _, err := netAgent.GetCmdChanResult(netAgentCh, ssntp.START) + if err != nil { + t.Fatal(err) + } + + // start CNCI agent + cnciAgent, err = testutil.NewSsntpTestClientConnection("CNCI Client", ssntp.CNCIAGENT, testutil.CNCIUUID) + if err != nil { + t.Fatal(err) + } + + controllerCh := controller.AddEventChan(ssntp.ConcentratorInstanceAdded) + + cnciAgent.SendConcentratorAddedEvent(testutil.CNCIInstanceUUID, testutil.TenantUUID, testutil.CNCIIP, testutil.CNCIMAC) + + _, err = controller.GetEventChanResult(controllerCh, ssntp.ConcentratorInstanceAdded) + if err != nil { + t.Fatal(err) + } +} + func TestStart(t *testing.T) { agentCh := agent.AddCmdChan(ssntp.START) @@ -377,6 +404,7 @@ func restartServer() error { controllerCh := controller.AddEventChan(ssntp.NodeConnected) netAgentCh := netAgent.AddEventChan(ssntp.NodeConnected) agentCh := agent.AddEventChan(ssntp.NodeConnected) + cnciAgentCh := cnciAgent.AddEventChan(ssntp.NodeConnected) server = configSchedulerServer() if server == nil { @@ -385,17 +413,29 @@ func restartServer() error { go server.ssntp.Serve(server.config, server) //go heartBeatLoop(server) ...handy for debugging - _, err := controller.GetEventChanResult(controllerCh, ssntp.NodeConnected) - if err != nil { - return err + if controller != nil { + _, err := controller.GetEventChanResult(controllerCh, ssntp.NodeConnected) + if err != nil { + return err + } } - _, err = netAgent.GetEventChanResult(netAgentCh, ssntp.NodeConnected) - if err != nil { - return err + if netAgent != nil { + _, err := netAgent.GetEventChanResult(netAgentCh, ssntp.NodeConnected) + if err != nil { + return err + } } - _, err = agent.GetEventChanResult(agentCh, ssntp.NodeConnected) - if err != nil { - return err + if agent != nil { + _, err := agent.GetEventChanResult(agentCh, ssntp.NodeConnected) + if err != nil { + return err + } + } + if cnciAgent != nil { + _, err := cnciAgent.GetEventChanResult(cnciAgentCh, ssntp.NodeConnected) + if err != nil { + return err + } } return nil } @@ -414,6 +454,39 @@ func TestReconnects(t *testing.T) { } } +func TestTenantAdded(t *testing.T) { + cnciAgentCh := cnciAgent.AddEventChan(ssntp.TenantAdded) + + go agent.SendTenantAddedEvent() + + _, err := cnciAgent.GetEventChanResult(cnciAgentCh, ssntp.TenantAdded) + if err != nil { + t.Fatal(err) + } +} + +func TestTenantRemoved(t *testing.T) { + cnciAgentCh := cnciAgent.AddEventChan(ssntp.TenantRemoved) + + go agent.SendTenantRemovedEvent() + + _, err := cnciAgent.GetEventChanResult(cnciAgentCh, ssntp.TenantRemoved) + if err != nil { + t.Fatal(err) + } +} + +func TestPublicIPAssigned(t *testing.T) { + controllerCh := controller.AddEventChan(ssntp.PublicIPAssigned) + + go cnciAgent.SendPublicIPAssignedEvent() + + _, err := controller.GetEventChanResult(controllerCh, ssntp.PublicIPAssigned) + if err != nil { + t.Fatal(err) + } +} + func waitForController(uuid string) { for { server.controllerMutex.Lock() @@ -503,15 +576,28 @@ func ssntpTestsSetup() error { func ssntpTestsTeardown() { // stop everybody - time.Sleep(1 * time.Second) - controller.Ssntp.Close() + var wg sync.WaitGroup + wg.Add(3) - time.Sleep(1 * time.Second) - netAgent.Ssntp.Close() + go func() { + controller.Ssntp.Close() + wg.Done() + }() - time.Sleep(1 * time.Second) - agent.Ssntp.Close() + go func() { + netAgent.Ssntp.Close() + wg.Done() + }() - time.Sleep(1 * time.Second) + go func() { + agent.Ssntp.Close() + wg.Done() + }() + + fmt.Println("Awaiting clients' shutdown") + wg.Wait() + fmt.Println("Got clients' shutdown") + fmt.Println("Awaiting server shutdown") server.ssntp.Stop() + fmt.Println("Got server shutdown") } diff --git a/ssntp/ssntp_test.go b/ssntp/ssntp_test.go index da3fcad3b..cacddeafc 100644 --- a/ssntp/ssntp_test.go +++ b/ssntp/ssntp_test.go @@ -21,14 +21,15 @@ import ( "encoding/asn1" "flag" "fmt" - . "github.com/01org/ciao/ssntp" - "github.com/01org/ciao/testutil" "io/ioutil" "os" "path" "sync" "testing" "time" + + . "github.com/01org/ciao/ssntp" + "github.com/01org/ciao/testutil" ) const tempCertPath = "/tmp/ssntp-test-certs" @@ -2465,6 +2466,160 @@ var ( payloadSize = flag.Int("payload", 1<<11, "Frames payload size") ) +func TestCommandStringer(t *testing.T) { + var stringTests = []struct { + cmd Command + expected string + }{ + {CONNECT, "CONNECT"}, + {START, "START"}, + {STOP, "STOP"}, + {STATS, "STATISTICS"}, + {EVACUATE, "EVACUATE"}, + {DELETE, "DELETE"}, + {RESTART, "RESTART"}, + {AssignPublicIP, "Assign public IP"}, + {ReleasePublicIP, "Release public IP"}, + {CONFIGURE, "CONFIGURE"}, + } + + for _, test := range stringTests { + str := test.cmd.String() + if str != test.expected { + t.Errorf("expected \"%s\", got \"%s\"", test.expected, str) + } + } +} + +func TestStatusStringer(t *testing.T) { + var stringTests = []struct { + sta Status + expected string + }{ + {CONNECTED, "CONNECTED"}, + {READY, "READY"}, + {FULL, "FULL"}, + {OFFLINE, "OFFLINE"}, + {MAINTENANCE, "MAINTENANCE"}, + } + + for _, test := range stringTests { + str := test.sta.String() + if str != test.expected { + t.Errorf("expected \"%s\", got \"%s\"", test.expected, str) + } + } +} + +func TestEventStringer(t *testing.T) { + var stringTests = []struct { + evt Event + expected string + }{ + {TenantAdded, "Tenant Added"}, + {TenantRemoved, "Tenant Removed"}, + {InstanceDeleted, "Instance Deleted"}, + {ConcentratorInstanceAdded, "Network Concentrator Instance Added"}, + {PublicIPAssigned, "Public IP Assigned"}, + {TraceReport, "Trace Report"}, + {NodeConnected, "Node Connected"}, + {NodeDisconnected, "Node Disconnected"}, + } + + for _, test := range stringTests { + str := test.evt.String() + if str != test.expected { + t.Errorf("expected \"%s\", got \"%s\"", test.expected, str) + } + } +} + +func TestErrorStringer(t *testing.T) { + var stringTests = []struct { + err Error + expected string + }{ + {InvalidFrameType, "Invalid SSNTP frame type"}, + {StartFailure, "Could not start instance"}, + {StopFailure, "Could not stop instance"}, + {ConnectionFailure, "SSNTP Connection failed"}, + {RestartFailure, "Could not restart instance"}, + {DeleteFailure, "Could not delete instance"}, + {ConnectionAborted, "SSNTP Connection aborted"}, + {InvalidConfiguration, "Cluster configuration is invalid"}, + } + + for _, test := range stringTests { + str := test.err.String() + if str != test.expected { + t.Errorf("expected \"%s\", got \"%s\"", test.expected, str) + } + } +} + +func TestRoleToDefaultCertName(t *testing.T) { + var stringTests = []struct { + r Role + expected string + }{ + {Controller, "/etc/pki/ciao/cert-Controller-localhost.pem"}, + {AGENT, "/etc/pki/ciao/cert-CNAgent-localhost.pem"}, + {CNCIAGENT, "/etc/pki/ciao/cert-CNCIAgent-localhost.pem"}, + {NETAGENT, "/etc/pki/ciao/cert-NetworkingAgent-localhost.pem"}, + {AGENT | NETAGENT, "/etc/pki/ciao/cert-CNAgent-NetworkingAgent-localhost.pem"}, + {SERVER, "/etc/pki/ciao/cert-Server-localhost.pem"}, + {SCHEDULER, "/etc/pki/ciao/cert-Scheduler-localhost.pem"}, + {UNKNOWN, ""}, + } + + for _, test := range stringTests { + certname := RoleToDefaultCertName(test.r) + if certname != test.expected { + t.Errorf("expected \"%s\", got \"%s\"", test.expected, certname) + } + } +} + +func TestRoleSet(t *testing.T) { + var stringTests = []struct { + r string + expected Role + }{ + {"unknown", UNKNOWN}, + {"server", SERVER}, + {"controller", Controller}, + {"agent", AGENT}, + {"netagent", NETAGENT}, + {"scheduler", SCHEDULER}, + {"cnciagent", CNCIAGENT}, + } + + for _, test := range stringTests { + var role Role + role.Set(test.r) + if role != test.expected { + t.Errorf("expected \"%x\", got \"%x\"", test.expected, role) + } + } + + var role Role + err := role.Set("asdf") + if err == nil { + t.Error("expected \"Unknown role\" error, got nil") + } + + role.Set("agent") + role.Set("netagent") + if role != AGENT|NETAGENT { + t.Error("didn't correctly sequantially assign role") + } + + role.Set("agent, netagent") + if role != AGENT|NETAGENT { + t.Error("didn't correctly multi-assign role") + } +} + func TestMain(m *testing.M) { flag.Parse() diff --git a/testutil/agent.go b/testutil/agent.go index bf14e4063..7425bccbe 100644 --- a/testutil/agent.go +++ b/testutil/agent.go @@ -278,12 +278,8 @@ func (client *SsntpTestClient) handleStart(payload []byte) Result { result.TenantUUID = cmd.Start.TenantUUID result.NodeUUID = client.UUID - if client.Role == ssntp.NETAGENT { - networking := cmd.Start.Networking - - client.sendConcentratorAddedEvent(cmd.Start.InstanceUUID, cmd.Start.TenantUUID, networking.VnicMAC) + if client.Role.IsNetAgent() { result.CNCI = true - return result } if client.StartFail == true { @@ -430,7 +426,7 @@ func (client *SsntpTestClient) CommandNotify(command ssntp.Command, frame *ssntp result = client.handleDelete(payload) default: - fmt.Printf("client unhandled command %s\n", command.String()) + fmt.Printf("client %s unhandled command %s\n", client.Role.String(), command.String()) } client.SendResultAndDelCmdChan(command, result) @@ -438,6 +434,28 @@ func (client *SsntpTestClient) CommandNotify(command ssntp.Command, frame *ssntp // EventNotify is an SSNTP callback stub for SsntpTestClient func (client *SsntpTestClient) EventNotify(event ssntp.Event, frame *ssntp.Frame) { + var result Result + + switch event { + case ssntp.TenantAdded: + var tenantAddedEvent payloads.EventTenantAdded + + err := yaml.Unmarshal(frame.Payload, &tenantAddedEvent) + if err != nil { + result.Err = err + } + case ssntp.TenantRemoved: + var tenantRemovedEvent payloads.EventTenantRemoved + + err := yaml.Unmarshal(frame.Payload, &tenantRemovedEvent) + if err != nil { + result.Err = err + } + default: + fmt.Printf("client %s unhandled event: %s\n", client.Role.String(), event.String()) + } + + client.SendResultAndDelEventChan(event, result) } // ErrorNotify is an SSNTP callback stub for SsntpTestClient @@ -534,20 +552,59 @@ func (client *SsntpTestClient) SendDeleteEvent(uuid string) { _, err = client.Ssntp.SendEvent(ssntp.InstanceDeleted, y) if err != nil { result.Err = err - fmt.Println(err) } } client.SendResultAndDelEventChan(ssntp.InstanceDeleted, result) } -func (client *SsntpTestClient) sendConcentratorAddedEvent(instanceUUID string, tenantUUID string, vnicMAC string) { +// SendTenantAddedEvent allows an SsntpTestClient to push an ssntp.TenantAdded event frame +func (client *SsntpTestClient) SendTenantAddedEvent() { + var result Result + + _, err := client.Ssntp.SendEvent(ssntp.TenantAdded, []byte(TenantAddedYaml)) + if err != nil { + result.Err = err + } + + client.SendResultAndDelEventChan(ssntp.TenantAdded, result) +} + +// SendTenantRemovedEvent allows an SsntpTestClient to push an ssntp.TenantRemoved event frame +func (client *SsntpTestClient) SendTenantRemovedEvent() { + var result Result + + _, err := client.Ssntp.SendEvent(ssntp.TenantRemoved, []byte(TenantRemovedYaml)) + if err != nil { + result.Err = err + } + + client.SendResultAndDelEventChan(ssntp.TenantRemoved, result) +} + +// SendPublicIPAssignedEvent allows an SsntpTestClient to push an ssntp.PublicIPAssigned event frame +func (client *SsntpTestClient) SendPublicIPAssignedEvent() { + var result Result + + _, err := client.Ssntp.SendEvent(ssntp.PublicIPAssigned, []byte(AssignedIPYaml)) + if err != nil { + result.Err = err + } + + client.SendResultAndDelEventChan(ssntp.PublicIPAssigned, result) +} + +// SendConcentratorAddedEvent allows an SsntpTestClient to push an ssntp.ConcentratorInstanceAdded event frame +func (client *SsntpTestClient) SendConcentratorAddedEvent(instanceUUID string, tenantUUID string, ip string, vnicMAC string) { + var result Result + evt := payloads.ConcentratorInstanceAddedEvent{ InstanceUUID: instanceUUID, TenantUUID: tenantUUID, - ConcentratorIP: "192.168.0.1", + ConcentratorIP: ip, ConcentratorMAC: vnicMAC, } + result.InstanceUUID = instanceUUID event := payloads.EventConcentratorInstanceAdded{ CNCIAdded: evt, @@ -555,13 +612,15 @@ func (client *SsntpTestClient) sendConcentratorAddedEvent(instanceUUID string, t y, err := yaml.Marshal(event) if err != nil { - return + result.Err = err + } else { + _, err = client.Ssntp.SendEvent(ssntp.ConcentratorInstanceAdded, y) + if err != nil { + result.Err = err + } } - _, err = client.Ssntp.SendEvent(ssntp.ConcentratorInstanceAdded, y) - if err != nil { - fmt.Println(err) - } + client.SendResultAndDelEventChan(ssntp.ConcentratorInstanceAdded, result) } func (client *SsntpTestClient) sendStartFailure(instanceUUID string, reason payloads.StartFailureReason) { diff --git a/testutil/client_server_test.go b/testutil/client_server_test.go index fd1a336c6..d02f6fcff 100644 --- a/testutil/client_server_test.go +++ b/testutil/client_server_test.go @@ -32,6 +32,7 @@ var server SsntpTestServer var controller *SsntpTestController var agent *SsntpTestClient var netAgent *SsntpTestClient +var cnciAgent *SsntpTestClient func TestSendAgentStatus(t *testing.T) { serverCh := server.AddStatusChan(ssntp.READY) @@ -55,17 +56,53 @@ func TestSendNetAgentStatus(t *testing.T) { } } +func TestCNCIStart(t *testing.T) { + serverCh := server.AddCmdChan(ssntp.START) + netAgentCh := netAgent.AddCmdChan(ssntp.START) + + go controller.Ssntp.SendCommand(ssntp.START, []byte(CNCIStartYaml)) + + _, err := server.GetCmdChanResult(serverCh, ssntp.START) + if err != nil { + t.Fatal(err) + } + _, err = netAgent.GetCmdChanResult(netAgentCh, ssntp.START) + if err != nil { + t.Fatal(err) + } + + serverCh = server.AddEventChan(ssntp.ConcentratorInstanceAdded) + controllerCh := controller.AddEventChan(ssntp.ConcentratorInstanceAdded) + + // start CNCI agent + cnciAgent, err = NewSsntpTestClientConnection("CNCI Client", ssntp.CNCIAGENT, CNCIUUID) + if err != nil { + t.Fatal(err) + } + + cnciAgent.SendConcentratorAddedEvent(CNCIInstanceUUID, TenantUUID, CNCIIP, CNCIMAC) + + _, err = server.GetEventChanResult(serverCh, ssntp.ConcentratorInstanceAdded) + if err != nil { + t.Fatal(err) + } + _, err = controller.GetEventChanResult(controllerCh, ssntp.ConcentratorInstanceAdded) + if err != nil { + t.Fatal(err) + } +} + func TestStart(t *testing.T) { serverCh := server.AddCmdChan(ssntp.START) agentCh := agent.AddCmdChan(ssntp.START) go controller.Ssntp.SendCommand(ssntp.START, []byte(StartYaml)) - _, err := agent.GetCmdChanResult(agentCh, ssntp.START) + _, err := server.GetCmdChanResult(serverCh, ssntp.START) if err != nil { t.Fatal(err) } - _, err = server.GetCmdChanResult(serverCh, ssntp.START) + _, err = agent.GetCmdChanResult(agentCh, ssntp.START) if err != nil { t.Fatal(err) } @@ -392,6 +429,22 @@ func TestDeleteFailure(t *testing.T) { } } +func TestTenantAdded(t *testing.T) { + serverCh := server.AddEventChan(ssntp.TenantAdded) + cnciAgentCh := cnciAgent.AddEventChan(ssntp.TenantAdded) + + go agent.SendTenantAddedEvent() + + _, err := server.GetEventChanResult(serverCh, ssntp.TenantAdded) + if err != nil { + t.Fatal(err) + } + _, err = cnciAgent.GetEventChanResult(cnciAgentCh, ssntp.TenantAdded) + if err != nil { + t.Fatal(err) + } +} + func stopServer() error { controllerCh := controller.AddEventChan(ssntp.NodeDisconnected) netAgentCh := netAgent.AddEventChan(ssntp.NodeDisconnected) @@ -418,25 +471,38 @@ func restartServer() error { controllerCh := controller.AddEventChan(ssntp.NodeConnected) netAgentCh := netAgent.AddEventChan(ssntp.NodeConnected) agentCh := agent.AddEventChan(ssntp.NodeConnected) + cnciAgentCh := cnciAgent.AddEventChan(ssntp.NodeConnected) StartTestServer(&server) //MUST be after StartTestServer becase the channels are initialized on start serverCh := server.AddEventChan(ssntp.NodeConnected) - _, err := controller.GetEventChanResult(controllerCh, ssntp.NodeConnected) - if err != nil { - return err + if controller != nil { + _, err := controller.GetEventChanResult(controllerCh, ssntp.NodeConnected) + if err != nil { + return err + } } - _, err = netAgent.GetEventChanResult(netAgentCh, ssntp.NodeConnected) - if err != nil { - return err + if netAgent != nil { + _, err := netAgent.GetEventChanResult(netAgentCh, ssntp.NodeConnected) + if err != nil { + return err + } } - _, err = agent.GetEventChanResult(agentCh, ssntp.NodeConnected) - if err != nil { - return err + if agent != nil { + _, err := agent.GetEventChanResult(agentCh, ssntp.NodeConnected) + if err != nil { + return err + } } - _, err = server.GetEventChanResult(serverCh, ssntp.NodeConnected) + if cnciAgent != nil { + _, err := cnciAgent.GetEventChanResult(cnciAgentCh, ssntp.NodeConnected) + if err != nil { + return err + } + } + _, err := server.GetEventChanResult(serverCh, ssntp.NodeConnected) if err != nil { return err } @@ -457,6 +523,38 @@ func TestReconnects(t *testing.T) { } } +func TestTenantRemoved(t *testing.T) { + serverCh := server.AddEventChan(ssntp.TenantRemoved) + cnciAgentCh := cnciAgent.AddEventChan(ssntp.TenantRemoved) + + go agent.SendTenantRemovedEvent() + + _, err := server.GetEventChanResult(serverCh, ssntp.TenantRemoved) + if err != nil { + t.Fatal(err) + } + _, err = cnciAgent.GetEventChanResult(cnciAgentCh, ssntp.TenantRemoved) + if err != nil { + t.Fatal(err) + } +} + +func TestPublicIPAssigned(t *testing.T) { + serverCh := server.AddEventChan(ssntp.PublicIPAssigned) + controllerCh := controller.AddEventChan(ssntp.PublicIPAssigned) + + go cnciAgent.SendPublicIPAssignedEvent() + + _, err := server.GetEventChanResult(serverCh, ssntp.PublicIPAssigned) + if err != nil { + t.Fatal(err) + } + _, err = controller.GetEventChanResult(controllerCh, ssntp.PublicIPAssigned) + if err != nil { + t.Fatal(err) + } +} + func TestMain(m *testing.M) { var err error diff --git a/testutil/controller.go b/testutil/controller.go index 22cfd4d57..922f23b8b 100644 --- a/testutil/controller.go +++ b/testutil/controller.go @@ -239,6 +239,17 @@ func (ctl *SsntpTestController) EventNotify(event ssntp.Event, frame *ssntp.Fram var result Result switch event { + // case ssntp.NodeConnected: handled by ConnectNotify() + // case ssntp.NodeDisconnected: handled by DisconnectNotify() + // case ssntp.TenantAdded: does not reach controller + // case ssntp.TenantRemoved: does not reach controller + case ssntp.PublicIPAssigned: + var publicIPAssignedEvent payloads.EventPublicIPAssigned + + err := yaml.Unmarshal(frame.Payload, &publicIPAssignedEvent) + if err != nil { + result.Err = err + } case ssntp.InstanceDeleted: var deleteEvent payloads.EventInstanceDeleted diff --git a/testutil/identity.go b/testutil/identity.go index 2d1003650..4b68970f7 100644 --- a/testutil/identity.go +++ b/testutil/identity.go @@ -267,6 +267,7 @@ type IdentityConfig struct { } // StartIdentityServer starts a fake keystone service for unit testing ciao. +// Caller must call Close() on the returned *httptest.Server. func StartIdentityServer(config IdentityConfig) *httptest.Server { id := httptest.NewServer(IdentityHandlers()) if id == nil { diff --git a/testutil/payloads.go b/testutil/payloads.go index 070c86975..51e834e0f 100644 --- a/testutil/payloads.go +++ b/testutil/payloads.go @@ -36,6 +36,9 @@ const InstancePrivateIP = "192.168.1.2" // VNICMAC is a test instance VNIC MAC address const VNICMAC = "aa:bb:cc:01:02:03" +// VNICUUID is a test instance VNIC UUID +const VNICUUID = "7f49d00d-1995-4156-8c79-5f5ab24ce138" + // TenantUUID is a test tenant UUID const TenantUUID = "2491851d-dce9-48d6-b83a-a717417072ce" @@ -90,6 +93,9 @@ const ImageUUID = "59460b8a-5f53-4e3e-b5ce-b71fed8c7e64" // InstanceUUID is an instance UUID for use in start/stop/restart/delete tests const InstanceUUID = "3390740c-dce9-48d6-b83a-a717417072ce" +// CNCIInstanceUUID is a CNCI instance UUID for use in start/stop/restart/delete tests +const CNCIInstanceUUID = "c6beb8b5-0bfc-43fd-9638-7dd788179fd8" + // NetAgentUUID is a network node UUID for coordinated tests const NetAgentUUID = "6be56328-92e2-4ecd-b426-8fe529c04e0c" @@ -138,7 +144,7 @@ const StartYaml = `start: // CNCIStartYaml is a sample CNCI workload START ssntp.Command payload for test cases const CNCIStartYaml = `start: - instance_uuid: ` + CNCIUUID + ` + instance_uuid: ` + CNCIInstanceUUID + ` image_uuid: ` + ImageUUID + ` fw_type: efi persistence: host @@ -153,6 +159,16 @@ const CNCIStartYaml = `start: - type: network_node value: 1 mandatory: true + networking: + vnic_mac: ` + VNICMAC + ` + vnic_uuid: ` + VNICUUID + ` + concentrator_uuid: ` + CNCIUUID + ` + concentrator_ip: ` + CNCIIP + ` + subnet: ` + TenantSubnet + ` + subnet_key: ` + SubnetKey + ` + subnet_uuid: "" + private_ip: "" + public_ip: false ` // PartialStartYaml is a sample minimal workload START ssntp.Command payload for test cases diff --git a/testutil/server.go b/testutil/server.go index 7f21f51b3..d4d6c8e99 100644 --- a/testutil/server.go +++ b/testutil/server.go @@ -29,9 +29,13 @@ import ( // SsntpTestServer is global state for the testutil SSNTP server type SsntpTestServer struct { - Ssntp ssntp.Server - clients []string - clientsLock *sync.Mutex + Ssntp ssntp.Server + + clients []string + clientsLock *sync.Mutex + netClients []string + netClientsLock *sync.Mutex + CmdChans map[ssntp.Command]chan Result CmdChansLock *sync.Mutex EventChans map[ssntp.Event]chan Result @@ -40,9 +44,6 @@ type SsntpTestServer struct { ErrorChansLock *sync.Mutex StatusChans map[ssntp.Status]chan Result StatusChansLock *sync.Mutex - - NetClients map[string]bool - NetClientsLock *sync.RWMutex } // AddCmdChan adds an ssntp.Command to the SsntpTestServer command channel @@ -204,9 +205,9 @@ func (server *SsntpTestServer) ConnectNotify(uuid string, role ssntp.Role) { server.clients = append(server.clients, uuid) case ssntp.NETAGENT: - server.NetClientsLock.Lock() - server.NetClients[uuid] = true - server.NetClientsLock.Unlock() + server.netClientsLock.Lock() + defer server.netClientsLock.Unlock() + server.netClients = append(server.netClients, uuid) } server.SendResultAndDelEventChan(ssntp.NodeConnected, result) @@ -216,20 +217,27 @@ func (server *SsntpTestServer) ConnectNotify(uuid string, role ssntp.Role) { func (server *SsntpTestServer) DisconnectNotify(uuid string, role ssntp.Role) { var result Result - server.clientsLock.Lock() - for index := range server.clients { - if server.clients[index] == uuid { - server.clients = append(server.clients[:index], server.clients[index+1:]...) - break + switch role { + case ssntp.AGENT: + server.clientsLock.Lock() + for index := range server.clients { + if server.clients[index] == uuid { + server.clients = append(server.clients[:index], server.clients[index+1:]...) + break + } } - } - server.clientsLock.Unlock() + server.clientsLock.Unlock() - server.NetClientsLock.Lock() - if server.NetClients[uuid] == true { - delete(server.NetClients, uuid) + case ssntp.NETAGENT: + server.netClientsLock.Lock() + for index := range server.netClients { + if server.netClients[index] == uuid { + server.netClients = append(server.netClients[:index], server.netClients[index+1:]...) + break + } + } + server.netClientsLock.Unlock() } - server.NetClientsLock.Unlock() server.SendResultAndDelEventChan(ssntp.NodeDisconnected, result) } @@ -340,14 +348,8 @@ func (server *SsntpTestServer) EventNotify(uuid string, event ssntp.Event, frame payload := frame.Payload switch event { - case ssntp.NodeConnected: - var connectEvent payloads.NodeConnected - - result.Err = yaml.Unmarshal(payload, &connectEvent) - case ssntp.NodeDisconnected: - var disconnectEvent payloads.NodeDisconnected - - result.Err = yaml.Unmarshal(payload, &disconnectEvent) + // case ssntp.NodeConnected: handled by ConnectNotify() + // case ssntp.NodeDisconnected: handled by DisconnectNotify() case ssntp.TraceReport: var traceEvent payloads.Trace @@ -363,7 +365,7 @@ func (server *SsntpTestServer) EventNotify(uuid string, event ssntp.Event, frame case ssntp.TenantRemoved: // forwards to CNCI via server.EventForward() case ssntp.PublicIPAssigned: - // forwards to CNCI via server.EventForward() + // forwards from CNCI Controller(s) via server.EventForward() default: fmt.Printf("server unhandled event %s\n", event.String()) } @@ -383,10 +385,6 @@ func getConcentratorUUID(event ssntp.Event, payload []byte) (string, error) { var ev payloads.EventTenantRemoved err := yaml.Unmarshal(payload, &ev) return ev.TenantRemoved.ConcentratorUUID, err - case ssntp.PublicIPAssigned: - var ev payloads.EventPublicIPAssigned - err := yaml.Unmarshal(payload, &ev) - return ev.AssignedIP.ConcentratorUUID, err } } @@ -406,20 +404,22 @@ func fwdEventToCNCI(event ssntp.Event, payload []byte) (ssntp.ForwardDestination func (server *SsntpTestServer) EventForward(uuid string, event ssntp.Event, frame *ssntp.Frame) ssntp.ForwardDestination { var err error var dest ssntp.ForwardDestination + var result Result switch event { case ssntp.TenantAdded: fallthrough case ssntp.TenantRemoved: - fallthrough - case ssntp.PublicIPAssigned: dest, err = fwdEventToCNCI(event, frame.Payload) } if err != nil { fmt.Println("server error parsing event yaml for forwarding") + result.Err = err } + server.SendResultAndDelEventChan(event, result) + return dest } @@ -461,47 +461,51 @@ func (server *SsntpTestServer) ErrorNotify(uuid string, error ssntp.Error, frame server.SendResultAndDelErrorChan(error, result) } -// CommandForward implements an SSNTP CommandForward callback for SsntpTestServer -func (server *SsntpTestServer) CommandForward(uuid string, command ssntp.Command, frame *ssntp.Frame) (dest ssntp.ForwardDestination) { - switch command { - case ssntp.START: - //TODO: move to a workload start function - var startCmd payloads.Start - var nn bool - - payload := frame.Payload +func (server *SsntpTestServer) handleStart(payload []byte) (dest ssntp.ForwardDestination) { + var startCmd payloads.Start + var nn bool - err := yaml.Unmarshal(payload, &startCmd) + err := yaml.Unmarshal(payload, &startCmd) - if err != nil { - return - } + if err != nil { + return + } - resources := startCmd.Start.RequestedResources + resources := startCmd.Start.RequestedResources - for i := range resources { - if resources[i].Type == payloads.NetworkNode { - nn = true - break - } + for i := range resources { + if resources[i].Type == payloads.NetworkNode { + nn = true + break } + } - if nn { - server.NetClientsLock.RLock() - for key := range server.NetClients { - dest.AddRecipient(key) - break - } - server.NetClientsLock.RUnlock() - return + if nn { + server.netClientsLock.Lock() + defer server.netClientsLock.Unlock() + if len(server.netClients) > 0 { + index := rand.Intn(len(server.netClients)) + dest.AddRecipient(server.netClients[index]) } - + } else { server.clientsLock.Lock() defer server.clientsLock.Unlock() if len(server.clients) > 0 { index := rand.Intn(len(server.clients)) dest.AddRecipient(server.clients[index]) } + } + + return dest +} + +// CommandForward implements an SSNTP CommandForward callback for SsntpTestServer +func (server *SsntpTestServer) CommandForward(uuid string, command ssntp.Command, frame *ssntp.Frame) (dest ssntp.ForwardDestination) { + payload := frame.Payload + + switch command { + case ssntp.START: + dest = server.handleStart(payload) case ssntp.EVACUATE: fallthrough case ssntp.STOP: @@ -521,6 +525,7 @@ func (server *SsntpTestServer) CommandForward(uuid string, command ssntp.Command // testutil.SsntpTestServer configuration with standard ssntp.FrameRorwardRules func StartTestServer(server *SsntpTestServer) { server.clientsLock = &sync.Mutex{} + server.netClientsLock = &sync.Mutex{} server.CmdChans = make(map[ssntp.Command]chan Result) server.CmdChansLock = &sync.Mutex{} @@ -534,9 +539,6 @@ func StartTestServer(server *SsntpTestServer) { server.StatusChans = make(map[ssntp.Status]chan Result) server.StatusChansLock = &sync.Mutex{} - server.NetClients = make(map[string]bool) - server.NetClientsLock = &sync.RWMutex{} - serverConfig := ssntp.Config{ CAcert: ssntp.DefaultCACert, Cert: ssntp.RoleToDefaultCertName(ssntp.SERVER), @@ -574,6 +576,10 @@ func StartTestServer(server *SsntpTestServer) { Operand: ssntp.DeleteFailure, Dest: ssntp.Controller, }, + { // all PublicIPAssigned events go to all Controllers + Operand: ssntp.PublicIPAssigned, + Dest: ssntp.Controller, + }, { // all START command are processed by the Command forwarder Operand: ssntp.START, CommandForward: server, @@ -602,10 +608,6 @@ func StartTestServer(server *SsntpTestServer) { Operand: ssntp.TenantRemoved, EventForward: server, }, - { // all PublicIPAssigned events are processed by the Event forwarder - Operand: ssntp.PublicIPAssigned, - EventForward: server, - }, }, }