Move all instances using transceivers to one TX domain (+whole bunch of bug fixes)

bittide-instances/src/Bittide/Instances/Hitl/LinkConfiguration.hs

@@ -177,19 +177,17 @@ transceiversStartAndObserve refClk sysClk rst myIndex rxNs rxPs miso =
         , clockPaths
         , rxNs
         , rxPs
-        , txDatas = myIndexTxN
+        , txDatas = repeat myIndexTx
         , rxReadys = repeat $ pure True
-        , txReadys = repeat $ pure True
+        , txStarts = repeat $ pure True
         }
 
   -- synchronizes the FPGA's stable index to the individual TX clock
   -- domains of the transceivers
-  myIndexTxN =
-    fmap (zeroExtend . pack . fromMaybe 0)
-      <$> zipWith
-        (xpmCdcStable sysClk myIndex)
-        transceivers.txClocks
-        transceivers.txResets
+  myIndexTx =
+    fmap
+      (zeroExtend . pack . fromMaybe 0)
+      (xpmCdcStable sysClk myIndex transceivers.txClock transceivers.txReset)
 
   -- check that all the received data matches with our expectations
   success =

bittide-instances/src/Bittide/Instances/Hitl/SwCcTopologies.hs

@@ -299,7 +299,7 @@ topologyTest refClk sysClk IlaControl{syncRst = rst, ..} rxNs rxPs miso cfg prog
         , rxNs
         , rxPs
         , txDatas = txCounters
-        , txReadys = repeat (pure True)
+        , txStarts = repeat (pure True)
         , rxReadys = repeat (pure True)
         }
 
@@ -373,7 +373,7 @@ topologyTest refClk sysClk IlaControl{syncRst = rst, ..} rxNs rxPs miso cfg prog
 
   callistoResult =
     callistoClockControlWithIla @LinkCount @CccBufferSize
-      (head transceivers.txClocks)
+      transceivers.txClock
       sysClk
       clockControlReset
       ccConfig
@@ -560,21 +560,20 @@ topologyTest refClk sysClk IlaControl{syncRst = rst, ..} rxNs rxPs miso cfg prog
       (domainDiffCounterExt sysClk)
       (orReset clockControlReset . unsafeFromActiveLow <$> othersNotInCCResetSync)
       transceivers.rxClocks
-      transceivers.txClocks
+      (repeat transceivers.txClock)
 
-  txAllStables = zipWith (xpmCdcSingle sysClk) transceivers.txClocks (repeat allStable1)
-  txResets2 =
-    zipWith
-      orReset
-      transceivers.txResets
-      (map unsafeFromActiveLow txAllStables)
+  txAllStable = xpmCdcSingle sysClk transceivers.txClock allStable1
+  txReset2 =
+    orReset
+      transceivers.txReset
+      (unsafeFromActiveLow txAllStable)
 
   txNotInCCResets :: Vec LinkCount (Signal GthTx Bool)
-  txNotInCCResets = go <$> transceivers.txClocks
+  txNotInCCResets = go <$> (repeat transceivers.txClock)
    where
     go txClk = unsafeSynchronizer sysClk txClk notInCCReset
 
-  txCounters = zipWith3 txCounter transceivers.txClocks txResets2 txNotInCCResets
+  txCounters = zipWith3 txCounter (repeat transceivers.txClock) (repeat txReset2) txNotInCCResets
   txCounter txClk txRst notInCCReset' = result
    where
     notInCCReset'' :: Signal GthTx (BitVector 1)
@@ -587,38 +586,38 @@ topologyTest refClk sysClk IlaControl{syncRst = rst, ..} rxNs rxPs miso cfg prog
   rxFifos =
     zipWith4
       go
-      transceivers.txClocks
+      (repeat transceivers.txClock)
       transceivers.rxClocks
-      txResets2
+      (repeat txReset2)
       transceivers.rxDatas
    where
     go = resettableXilinxElasticBuffer @FifoSize @_ @_ @(Maybe (BitVector 64))
 
   (_fillLvls, fifoUnderflowsTx, fifoOverflowsTx, _ebMode, mRxCntrs) = unzip5 rxFifos
-  rxCntrs = zipWith3 go transceivers.txClocks txNotInCCResets mRxCntrs
+  rxCntrs = zipWith go txNotInCCResets mRxCntrs
    where
-    go txClk txRst = regMaybe txClk (unsafeFromActiveLow txRst) enableGen rxCounterStartUgn
+    go txRst = regMaybe transceivers.txClock (unsafeFromActiveLow txRst) enableGen rxCounterStartUgn
 
   fifoOverflowsFree :: Vec LinkCount (Signal Basic125 Overflow)
-  fifoOverflowsFree = zipWith (`xpmCdcSingle` sysClk) transceivers.txClocks fifoOverflowsTx
+  fifoOverflowsFree = map (xpmCdcSingle transceivers.txClock sysClk) fifoOverflowsTx
   fifoUnderflowsFree :: Vec LinkCount (Signal Basic125 Underflow)
-  fifoUnderflowsFree = zipWith (`xpmCdcSingle` sysClk) transceivers.txClocks fifoUnderflowsTx
+  fifoUnderflowsFree = map (xpmCdcSingle transceivers.txClock sysClk) fifoUnderflowsTx
 
   ugns1 :: Vec LinkCount (Signal GthTx (BitVector 64))
   ugns1 = zipWith (-) txCounters rxCntrs
   -- see NOTE [magic start values]
 
   ugns2 :: Vec LinkCount (Signal Basic125 (BitVector 64))
-  ugns2 = zipWith3 go transceivers.txClocks othersNotInCCResetSync ugns1
+  ugns2 = zipWith go othersNotInCCResetSync ugns1
    where
-    go txClk enaSig =
+    go enaSig =
       regEn
         sysClk
         clockControlReset
         enableGen
         rxCounterStartUgn
         enaSig
-        . xpmCdcArraySingle txClk sysClk
+        . xpmCdcArraySingle transceivers.txClock sysClk
 
   ugnsStable :: Vec LinkCount (Signal Basic125 Bool)
   ugnsStable = go <$> ugns2

bittide-instances/src/Bittide/Instances/Hitl/Transceivers.hs

@@ -50,16 +50,15 @@ a non-zero start value, as a regression test for a bug where the transceivers
 would not come up if the counters started at zero.
 -}
 counterStart :: BitVector 64
-counterStart = 0xDEAD_BEEF_CA55_E77E
+counterStart = 0xDEAD_BEEF_0000_0000
 
 -- | A counter starting at 'counterStart'
 counter ::
   (KnownDomain dom) =>
   Clock dom ->
   Reset dom ->
-  Signal dom Bool ->
   Signal dom (BitVector 64)
-counter clk rst ena = let c = register clk rst (toEnable ena) counterStart (c + 1) in c
+counter clk rst = let c = register clk rst enableGen counterStart (c + 1) in c
 
 {- | Expect a counter starting at 'counterStart' and incrementing by one on each
 cycle.
@@ -128,13 +127,18 @@ goTransceiversUpTest fpgaIndex refClk sysClk rst rxNs rxPs miso =
   -- Transceiver setup
   gthAllReset = unsafeFromActiveLow spiDone
 
-  -- Send counters
-  counters =
-    zipWith3
-      counter
-      transceivers.txClocks
-      transceivers.txResets
-      transceivers.txSamplings
+  txCounter =
+    counter
+      transceivers.txClock
+      -- We use a single counter for all transmit channels and we start them all
+      -- at the same time. This way, the other side will always receive 'counterStart'
+      -- as the first value and can verify all samples after that by incrementing
+      -- by one each cycle. Note that it doesn't really matter whether we pick
+      -- (.||.) or (.&&.) here, as all links should start transmitting at the
+      -- same time. I.e., txSamplings should all flip at the same time. Still, we
+      -- picked (.||.) to catch situations where some links start transmitting
+      -- earlier than others.
+      (unsafeFromActiveLow (fold (.||.) transceivers.txSamplings))
 
   expectCounterError =
     zipWith3
@@ -144,11 +148,13 @@ goTransceiversUpTest fpgaIndex refClk sysClk rst rxNs rxPs miso =
       transceivers.rxDatas
 
   expectCounterErrorSys =
-    fmap and
+    fmap or
       $ bundle
       $ zipWith (.&&.) transceivers.linkUps
       $ zipWith (`xpmCdcSingle` sysClk) transceivers.rxClocks expectCounterError
 
+  txStart = fold (.&&.) transceivers.txReadys
+
   transceivers =
     transceiverPrbsN
       @GthTx
@@ -166,8 +172,8 @@ goTransceiversUpTest fpgaIndex refClk sysClk rst rxNs rxPs miso =
         , clockPaths
         , rxNs
         , rxPs
-        , txDatas = counters
-        , txReadys = repeat (pure True)
+        , txDatas = repeat txCounter
+        , txStarts = repeat txStart
         , rxReadys = repeat (pure True)
         }
 

bittide-shake/exe/Main.hs

@@ -16,7 +16,11 @@ module Main where
 
 import Prelude
 
-import Bittide.Hitl (HitlTestGroup (..), hwTargetRefsFromHitlTestGroup)
+import Bittide.Hitl (
+  HitlTestGroup (..),
+  TestStepResult (..),
+  hwTargetRefsFromHitlTestGroup,
+ )
 import Bittide.Instances.Hitl.Tests (ClashTargetName, hitlTests)
 import Clash.DataFiles (tclConnector)
 import Clash.Shake.Extra
@@ -556,18 +560,23 @@ main = do
                 need [testExitCodePath]
                 exitCode <- read <$> readFile' testExitCodePath
                 when (isJust (mPostProc =<< targetTest)) $ do
-                  _ <- liftIO $ (fromJust $ mPostProc =<< targetTest) ilaDataDir exitCode
-                  pure ()
+                  res <- liftIO $ (fromJust $ mPostProc =<< targetTest) ilaDataDir exitCode
+                  checkTestStep res
                 unless (exitCode == ExitSuccess) $ do
                   liftIO $ exitWith exitCode
 
               when (isJust (mPostProc =<< targetTest)) $ do
                 phony (entityName targetName <> ":post-process") $ do
                   need [testExitCodePath]
                   exitCode <- read <$> readFile' testExitCodePath
-                  _ <- liftIO $ (fromJust (mPostProc =<< targetTest)) ilaDataDir exitCode
-                  pure ()
+                  res <- liftIO $ (fromJust (mPostProc =<< targetTest)) ilaDataDir exitCode
+                  checkTestStep res
 
     if null shakeTargets
       then rules
       else want shakeTargets >> withoutActions rules
+
+checkTestStep :: (MonadFail m, HasCallStack) => TestStepResult a -> m a
+checkTestStep res = case res of
+  TestStepFailure err -> error err
+  TestStepSuccess x -> return x

bittide-tools/clockcontrol/plot/Main.hs

@@ -113,6 +113,8 @@ import System.IO (
   hPutStr,
   hSetBuffering,
   openFile,
+  stderr,
+  stdout,
   withFile,
  )
 import Text.Read (readMaybe)
@@ -674,6 +676,8 @@ plotTest refDom testDir cfg dir globalOutDir = do
         -- Fail if clocks did not start at their set offsets. We purposely fail
         -- after generating the report, because the report generation is very
         -- useful for debugging.
+        hFlush stdout
+        hFlush stderr
         maybe (return ()) die maybeError
       _ -> die "Empty topology"
     _ -> die "Topology is larger than expected"