staged-flow 4/6: regenerate missing or stale Verilog when linking

src/comp/Error.hs

@@ -1141,6 +1141,7 @@ data ErrMsg =
 
         -- ABin (.ba) file issues
         | WExtraABinFiles [String]
+        | WNoABinForVerilogRegen String
         | EMissingABinModFile String (Maybe String)
         | EMissingABinForeignFuncFile String String
         | EMultipleABinFilesForName String [String]
@@ -4143,6 +4144,12 @@ getErrorText (WExtraABinFiles filenames) =
     (System 39, empty,
      s2par ("The following elaboration files were not used in the design:") $$
      nest 2 (sepList (map text filenames) comma))
+getErrorText (WNoABinForVerilogRegen topmod) =
+    (System 99, empty,
+     s2par ("No elaboration file (.ba) was found for module " ++
+            ishow topmod ++ " or one of its submodules, so BSC cannot " ++
+            "check whether the generated Verilog is up to date; linking " ++
+            "will use the Verilog files found on the search path."))
 getErrorText (EMissingABinModFile module_name mparent) =
     (System 40, empty,
      case (mparent) of

src/comp/SimFileUtils.hs

@@ -13,8 +13,9 @@ import Version(bscVersionStr)
 import FileNameUtil
 import ErrorUtil(internalError)
 
+import StaleUtils(allFreshVs)
+
 import System.Posix.Files
-import System.Posix.Types(EpochTime)
 import System.IO(openFile, hGetContents, hClose, IOMode(..))
 import Control.Monad(filterM)
 import Control.Exception(bracketOnError)
@@ -23,14 +24,6 @@ import qualified Data.Map as M
 
 -- import Debug.Trace(traceM)
 
-getModTime :: FilePath -> IO (Maybe EpochTime)
-getModTime f =
-    do ok <- fileExist f
-       if ok
-        then do s <- getFileStatus f
-                return $ Just (modificationTime s)
-        else return Nothing
-
 -- Settings that make an object unreusable under a different setting (isStale
 -- checks these before reuse).  blockCodegen (-c mode) is omitted: its output
 -- equals the submodule form (see DEVELOP.md).  "top" marks top form (the
@@ -66,18 +59,12 @@ isStale flags prefix ba_map top_pkg pkg =
               Nothing        -> internalError $ "isStale: unknown package " ++ name
               (Just ba_file) -> do h_file <- genFileName mkHName (cdir flags) "" name
                                    o_file <- genFileName mkObjName (cdir flags) "" name
-                                   ba_time <- getModTime ba_file
-                                   h_time <- getModTime h_file
-                                   obj_time <- getModTime o_file
-                                   let stale_time =
-                                        case (ba_time, h_time, obj_time) of
-                                          (Just t1, Just t2, Just t3) -> (t2 < t1) || (t3 < t1)
-                                          _                           -> True
+                                   fresh_time <- allFreshVs ba_file [h_file, o_file]
                                    cg_opt <- readCodeGenOptionDescr h_file
                                    let is_top = (sp_name pkg) == top_pkg
                                        cg_tgt = Just ("/* " ++ (codeGenOptionDescr flags is_top) ++ " */")
                                        stale_options = cg_opt /= cg_tgt
-                                   return $ stale_time || stale_options
+                                   return $ (not fresh_time) || stale_options
 
 remove_stale :: M.Map String [String] -> [String] -> [String] -> [String]
 remove_stale _     []   _      = []

src/comp/StaleUtils.hs

@@ -0,0 +1,30 @@
+module StaleUtils ( getModTime, allFreshVs ) where
+
+import System.Posix.Files
+import System.Posix.Types(EpochTime)
+
+-- Shared primitives for the generated-file staleness checks: the -u
+-- recompilation check (Depend), Bluesim object reuse (SimFileUtils) and
+-- Verilog reuse at link (VFileUtils) all decide "regenerate or reuse"
+-- by comparing a source artifact's time against its products'.  The
+-- conventions live here so the checks agree: a missing product is never
+-- fresh, and equal timestamps are fresh.
+
+-- modification time of a file, or Nothing if it does not exist
+getModTime :: FilePath -> IO (Maybe EpochTime)
+getModTime f =
+    do ok <- fileExist f
+       if ok
+        then do s <- getFileStatus f
+                return $ Just (modificationTime s)
+        else return Nothing
+
+-- are all the products at least as new as the source?
+-- (a missing product, or a missing source, is not fresh)
+allFreshVs :: FilePath -> [FilePath] -> IO Bool
+allFreshVs source products = do
+    msrc <- getModTime source
+    mprods <- mapM getModTime products
+    let fresh (Just st) (Just pt) = pt >= st
+        fresh _         _         = False
+    return $ all (fresh msrc) mprods

src/comp/VFileUtils.hs

@@ -0,0 +1,42 @@
+module VFileUtils ( partitionStaleVerilogMods ) where
+
+import Flags(Flags(..))
+import ABin(ABinModInfo(..))
+import ASyntax(apkg_name)
+import Id(unQualId, getIdString)
+import FileNameUtil(genFileName, mkVName, getFullFilePath, getRelativeFilePath)
+import StaleUtils(allFreshVs)
+import VFileName
+
+import Data.Either(partitionEithers)
+
+-- Check whether generated Verilog files are up to date with respect to
+-- their elaborated (.ba) files.  The Verilog analogue of SimFileUtils,
+-- built on the same StaleUtils conventions, with deliberate differences:
+--   * no transitive invalidation: a parent's .v refers to child modules
+--     by name only, so a stale child never invalidates a fresh parent
+--   * no version check: a .ba that loads is current-version by
+--     construction (decodeABin rejects other versions when it is read)
+--   * no options descriptor (for now): a .v generated under different
+--     codegen flags is reused as long as it is newer than the .ba
+
+-- Split modules into (stale: regenerate, fresh: reuse the .v).  A
+-- module's .v is stale when it is missing from the location BSC would
+-- write it (per -vdir and the prefix) or is older than its .ba.  Fresh
+-- modules are returned as (full path for the simulator, relative name
+-- for reporting), mirroring writeVerilog's two uses of the file name.
+partitionStaleVerilogMods :: Flags -> String
+                          -> [(String, ABinModInfo)]
+                          -> IO ([(String, ABinModInfo)], [(VFileName, String)])
+partitionStaleVerilogMods flags prefix abmis = do
+    let checkOne (bafile, abmi) = do
+          let modstr = getIdString (unQualId (apkg_name (abmi_apkg abmi)))
+          vName_init <- genFileName mkVName (vdir flags) prefix modstr
+          let vName = getFullFilePath vName_init
+              vNameRel = getRelativeFilePath vName_init
+          fresh <- allFreshVs bafile [vName]
+          return $ if fresh
+                   then Right (VFileName vName, vNameRel)
+                   else Left (bafile, abmi)
+    results <- mapM checkOne abmis
+    return (partitionEithers results)

src/comp/bsc.hs

@@ -15,7 +15,7 @@ import System.Directory(getDirectoryContents, doesFileExist, getCurrentDirectory
 import System.Time(getClockTime, ClockTime(TOD)) -- XXX: from old-time package
 import Data.Char(isSpace, toLower, ord)
 import Data.List(intersect, nub, partition, intersperse, sort,
-            isPrefixOf, isSuffixOf, unzip5, intercalate, foldl')
+            isPrefixOf, isSuffixOf, unzip5, intercalate, foldl', tails)
 import Data.Time.Clock.POSIX(getPOSIXTime)
 import Data.Maybe(isJust, isNothing)
 import Numeric(showOct)
@@ -143,6 +143,7 @@ import SimCOpt(simCOpt)
 import SimBlocksToC(simBlocksToC)
 import SystemCWrapper(checkSystemCIfc, wrapSystemC)
 import SimFileUtils(analyzeBluesimDependencies)
+import VFileUtils(partitionStaleVerilogMods)
 import Verilog(VProgram(..), vGetMainModName, getVeriInsts)
 import Depend
 import Version(bscVersionStr, copyright, buildnum)
@@ -1597,6 +1598,11 @@ simLink errh flags toplevel afilenames cfilenames = do
 
 -- Reuse a Bluesim generated object file
 -- returns the name of the object file being reused
+-- report that a generated Verilog file was reused rather than regenerated
+reuseVerilogFile :: Flags -> String -> IO ()
+reuseVerilogFile flags vNameRel =
+    unless (quiet flags) $ putStrLnF ("Verilog file reused: " ++ vNameRel)
+
 reuseBluesimCFile :: Flags -> String -> IO String
 reuseBluesimCFile flags oName = do
     -- show is used for quoting
@@ -2046,11 +2052,12 @@ vLink errh flags topmod_name vfilenames0 afilenames cfilenames = do
                           [(cmdPosition,
                             EMultipleABinFilesForName link_name file_names)]
 
-            -- XXX Until we allow re-generation of Verilog files,
-            -- XXX the module .ba files are unused
-            when (not (null (mod_abis))) $
-                bsWarning errh
-                    [(cmdPosition, WExtraABinFiles (map fst mod_abis))]
+            -- without the design's .ba hierarchy, the staleness of
+            -- generated Verilog cannot be checked; warn and use the .v
+            -- files as found (module .ba files given explicitly on the
+            -- command line are still regenerated from, below)
+            bsWarning errh
+                [(cmdPosition, WNoABinForVerilogRegen topmod_name)]
 
             let ffuncs = [ abffi_foreign_func abfi
                            | (_, (ABinForeignFunc abfi _)) <- ffunc_abis ]
@@ -2073,17 +2080,32 @@ vLink errh flags topmod_name vfilenames0 afilenames cfilenames = do
     t <- dump errh flags t DFreadelab dumpnames
              (map (pfpString . ff_name) ffuncs ++ map fst mod_abmis)
 
-{-
-    -- generate Verilog for the module abis
-    -- XXX only if the verilog doesn't exist or is older
-    -- XXX print a message about reusing a file?
-    (t, gen_vfilenames) <-
-        if (updCheck flags)
-        then vGenMods t flags mod_abmis
-        else return (t, [])
-    let vfilenames = vfilenames0  ++ gen_vfilenames
--}
-    let vfilenames = vfilenames0
+    -- Regenerate any module .v that is missing or older than its .ba, and
+    -- reuse the rest, so a link is self-sufficient regardless of which
+    -- subset was pre-generated (with -c or an earlier compile).  A module
+    -- whose .v was given explicitly on the command line is the user's to
+    -- provide and is never regenerated.  File arguments can be glob
+    -- patterns (bsc forwards them to the simulator's shell unexpanded, so
+    -- "*.v" has always meant "the user provides every module"); match
+    -- module names against the patterns rather than comparing literally.
+    let vfile_pats = [ baseName (dropSuf (vfnString vfn))
+                     | vfn <- vfilenames0 ]
+        -- glob match with * and ? (the shell metacharacters that can
+        -- appear in a forwarded filename argument)
+        globMatch ('*':ps) s = any (globMatch ps) (tails s)
+        globMatch ('?':ps) (_:cs) = globMatch ps cs
+        globMatch (p:ps) (c:cs) = (p == c) && globMatch ps cs
+        globMatch [] [] = True
+        globMatch _ _ = False
+        abmiModName abmi = getIdString (unQualId (apkg_name (abmi_apkg abmi)))
+        checked_abmis = [ p | p@(_, abmi) <- mod_abmis
+                            , not (any (`globMatch` abmiModName abmi)
+                                       vfile_pats) ]
+    (stale_abmis, reused_vs) <-
+        partitionStaleVerilogMods flags prefix checked_abmis
+    mapM_ (reuseVerilogFile flags . snd) reused_vs
+    (t, gen_vfilenames) <- vGenMods errh flags t stale_abmis
+    let vfilenames = vfilenames0 ++ map fst reused_vs ++ gen_vfilenames
 
     -- generate files for the foreign functions
     start flags DFcompileVPI

testsuite/bsc.driver/no_filenames/vlog_build.out.expected

@@ -1 +1,7 @@
+Verilog file reused: vd/mkSub3.v
+Verilog file reused: vd/mkMid2.v
+Verilog file reused: vd/mkSub2.v
+Verilog file reused: vd/mkSub1.v
+Verilog file reused: vd/mkMid1.v
+Verilog file reused: vd/mkTop.v
 Verilog binary file created: mkTop.vexe

testsuite/bsc.verilog/vlink_regen/Makefile

@@ -0,0 +1,5 @@
+# for "make clean" to work everywhere
+
+CONFDIR = $(realpath ../..)
+
+include $(CONFDIR)/clean.mk

testsuite/bsc.verilog/vlink_regen/VRegen.bsv

@@ -0,0 +1,21 @@
+// Fixtures for link-time regeneration of Verilog from .ba
+
+(* synthesize *)
+module mkRSub(Reg#(Bit#(8)));
+   Reg#(Bit#(8)) rg <- mkReg(0);
+   return rg;
+endmodule
+
+(* synthesize *)
+module sysVRegenTb(Empty);
+   Reg#(Bit#(8)) sub <- mkRSub;
+   Reg#(Bit#(4)) n <- mkReg(0);
+   rule bump;
+      sub <= sub + 3;
+      n <= n + 1;
+      if (n == 4) begin
+         $display("sub = %0d", sub);
+         $finish(0);
+      end
+   endrule
+endmodule

testsuite/bsc.verilog/vlink_regen/sysVRegenTb.out.expected

@@ -0,0 +1 @@
+sub = 12

testsuite/bsc.verilog/vlink_regen/vlink_regen.exp

@@ -0,0 +1,107 @@
+#
+# Tests for link-time regeneration of Verilog from .ba files:
+# "bsc -verilog -e top" regenerates any module .v that is missing or
+# older than its .ba, reuses the rest, and warns (and links with the .v
+# files as found) when the design's .ba hierarchy is unavailable.
+#
+
+# run "bsc -verilog -e" capturing output; expect success
+proc vlink_run { toplevel output {files ""} } {
+    set opts "-no-show-timestamps -no-show-version -verilog -e $toplevel -o $toplevel.vexe $files"
+    if [ gen_basic_output $opts $output ] {
+        pass "link $toplevel ($output)"
+    } else {
+        fail "link $toplevel should succeed ($output)"
+    }
+}
+
+# -------------------------
+# Elaborate the design (writes .v and .ba for both modules)
+
+compile_verilog_pass VRegen.bsv sysVRegenTb
+
+# -------------------------
+# A fresh link reuses both generated .v files
+
+vlink_run sysVRegenTb vlink1.bsc-out
+find_regexp vlink1.bsc-out {Verilog file reused: mkRSub\.v}
+find_regexp vlink1.bsc-out {Verilog file reused: sysVRegenTb\.v}
+find_regexp_fail vlink1.bsc-out {Verilog file created}
+
+# -------------------------
+# A deleted .v is regenerated at link, and the result simulates
+
+erase mkRSub.v
+vlink_run sysVRegenTb vlink2.bsc-out
+find_regexp vlink2.bsc-out {Verilog file created: mkRSub\.v}
+find_regexp vlink2.bsc-out {Verilog file reused: sysVRegenTb\.v}
+files_exist { mkRSub.v }
+if { $vtest == 1 } {
+    sim_verilog sysVRegenTb
+    compare_file sysVRegenTb.out sysVRegenTb.out.expected
+}
+
+# -------------------------
+# A .v older than its .ba is regenerated; others are reused
+
+exec sleep 1
+exec touch [file join [absolute $srcdir] $subdir mkRSub.ba]
+vlink_run sysVRegenTb vlink3.bsc-out
+find_regexp vlink3.bsc-out {Verilog file created: mkRSub\.v}
+find_regexp vlink3.bsc-out {Verilog file reused: sysVRegenTb\.v}
+
+# -------------------------
+# Nothing stale: everything is reused, and the .v files are untouched
+
+copy mkRSub.v mkRSub-before.v
+vlink_run sysVRegenTb vlink4.bsc-out
+find_regexp_fail vlink4.bsc-out {Verilog file created}
+compare_file mkRSub.v mkRSub-before.v
+
+# -------------------------
+# A .v given explicitly on the command line is never regenerated,
+# even when it is older than its .ba
+
+exec sleep 1
+exec touch [file join [absolute $srcdir] $subdir mkRSub.ba]
+vlink_run sysVRegenTb vlink5.bsc-out {mkRSub.v}
+find_regexp_fail vlink5.bsc-out {Verilog file created: mkRSub\.v}
+
+# (bring it back up to date for any later steps)
+vlink_run sysVRegenTb vlink5b.bsc-out
+
+# -------------------------
+# The .ba check is all-or-nothing per design: if any module's .ba is
+# missing (here a submodule's, while the top's is present), the link
+# warns and uses the .v files as found -- it never half-regenerates a
+# design it cannot fully see -- and still succeeds
+
+erase mkRSub.ba
+vlink_run sysVRegenTb vlink7.bsc-out
+find_n_strings vlink7.bsc-out {Warning: Command line: (S0099)} 1
+find_regexp_fail vlink7.bsc-out {Verilog file created}
+find_regexp_fail vlink7.bsc-out {Verilog file reused}
+
+# ... and the same from the other direction: the top's .ba missing
+# while the submodule's is present
+
+compile_verilog_pass VRegen.bsv sysVRegenTb
+erase sysVRegenTb.ba
+vlink_run sysVRegenTb vlink8.bsc-out
+find_n_strings vlink8.bsc-out {Warning: Command line: (S0099)} 1
+find_regexp_fail vlink8.bsc-out {Verilog file created}
+find_regexp_fail vlink8.bsc-out {Verilog file reused}
+
+# -------------------------
+# With no .ba hierarchy available, the link warns and uses the .v
+# files as found (the pre-regeneration behavior)
+
+erase sysVRegenTb.ba
+erase mkRSub.ba
+vlink_run sysVRegenTb vlink6.bsc-out
+find_n_strings vlink6.bsc-out {Warning: Command line: (S0099)} 1
+find_regexp_fail vlink6.bsc-out {Verilog file created}
+if { $vtest == 1 } {
+    sim_verilog sysVRegenTb
+    compare_file sysVRegenTb.out sysVRegenTb.out.expected
+}