Merge pull request #37 from JuliaGeodynamics/bk-read-logfiles

Read LaMEM logfiles

Project.toml

@@ -15,8 +15,8 @@ Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
 [compat]
-DocStringExtensions = "0.9"
 DelimitedFiles = "1"
+DocStringExtensions = "0.9"
 GeoParams = "0.4, 0.5"
 GeophysicalModelGenerator = "0.4 - 0.6"
 Glob = "1"

src/LaMEM.jl

@@ -12,7 +12,7 @@ using .IO_functions
 export Read_LaMEM_PVTR_File, Read_LaMEM_PVTS_File, Read_LaMEM_PVTU_File
 export Read_LaMEM_simulation, Read_LaMEM_timestep, Read_LaMEM_fieldnames
 export PassiveTracer_Time
-export clean_directory, changefolder, read_phase_diagram
+export clean_directory, changefolder, read_phase_diagram, read_LaMEM_logfile
 
 # Functions 
 include("Run.jl")

src/utils_Run.jl

@@ -1,4 +1,4 @@
-export remove_popup_messages_mac, show_paths_LaMEM
+export remove_popup_messages_mac, show_paths_LaMEM, read_LaMEM_logfile
 
 """
     remove_popup_messages_mac()
@@ -70,3 +70,158 @@ function show_paths_LaMEM()
 
     return nothing
 end
+
+"""
+This reads a LaMEM logfile (provided it was run with "-log_view") and collects key results from it; 
+mostly for scalability tests on HPC machines. It returns a markdown summary
+"""
+function read_LaMEM_logfile(Filename::String; ID=nothing, header=true)
+
+    # Read file as vector of strings
+    f = open(Filename)
+    lines = readlines(f)
+    close(f)
+
+    # Extract information from logfile
+    Cores       = Int64(extract_info_logfile(lines, "Total number of cpu                  :",LaMEM=false, entry=1))
+    FineGrid    = extract_info_logfile(lines, "Fine grid cells [nx, ny, nz]         :")
+    CoarseGrid  = extract_info_logfile(lines, "Global coarse grid [nx,ny,nz] :")
+    Levels      = parse(Int64,extract_info_logfile(lines, "Number of multigrid levels    :"))
+
+    TotalTime    = extract_info_logfile(lines, "Time (sec):", LaMEM=false)
+    CoarseTime   = extract_info_logfile(lines, "MGSmooth Level 0", LaMEM=false, entry=3)
+    SNES         = Int64(extract_info_logfile(lines, "SNESSolve", LaMEM=false, entry=1))
+    KSP          = Int64(extract_info_logfile(lines, "KSPSolve", LaMEM=false, entry=1))
+    RedFactor    = Int64(extract_info_logfile(lines, "-crs_pc_telescope_reduction_factor", LaMEM=false, entry=1))
+    CoaCores     = Int64(Cores/RedFactor)
+
+    # Retrieve memory usage if we have system with slurm (use seff)
+    if isnothing(ID)
+        ID = split(split(Filename,".")[1],"_")[end]
+        ID = parse(Int64,ID)
+    end
+    lines_mem = execute_command(ID); # run code
+    if !isnothing(lines_mem)
+        Memory_Gb  = extract_info_logfile(lines_mem, "Memory Utilized:", LaMEM=false, entry=1)
+        #Memory_Gb = extract_info_logfile(lines_mem, "Memory Utilized:", LaMEM=false, entry=5)
+
+        Nodes      = Int64(extract_info_logfile(lines_mem, "Nodes:", LaMEM=false, entry=1))
+        MemNode_Gb = Memory_Gb/Nodes
+    else
+        Memory_Gb = "-"
+        Nodes = "-"
+        MemNode_Gb = "-"
+    end
+
+
+    # print as Markdown table
+    table = (; FineGrid, Cores, Nodes, CoarseGrid, CoaCores, Levels, SNES, KSP, TotalTime, CoarseTime, MemNode_Gb, Filename) 
+    print_table_markdown(table, header=header)
+
+    return nothing
+end
+
+
+function read_LaMEM_logfile(Filename::Vector{String}; ID=nothing)
+    read_LaMEM_logfile(Filename[1]; ID=ID, header=true)
+    for i = 2: length(Filename) 
+        read_LaMEM_logfile(Filename[i]; ID=ID, header=false)
+    end
+end
+
+add_str(str, keyword, pad) = str*"| $(rpad(keyword,pad))"
+
+"""
+
+    value = extract_info_logfile(lines, keyword::String; entry=1, LaMEM=true)
+
+Internal function to extract information from the logfile
+
+Note that the LaMEM keywords should contain ":" at the end, while the PETSc keywords should not, but we have to indicate the entry number for the PETSc keywords.
+Example LaMEM keyword:
+```julia
+julia> val = extract_info_logfile(lines, "Fine grid cells [nx, ny, nz]         :")
+"[512, 256, 256]"
+```
+
+Example PETSc keyword:
+```julia
+julia> coarse_grid_solve = extract_info_logfile(lines, "MGSmooth Level 0", LaMEM=false, entry=3)
+9.9174
+```
+"""
+function extract_info_logfile(lines, keyword::String; entry=1, LaMEM=true)
+    # find the line with the keyword
+    idx = findlast(x->occursin(keyword,x), lines)
+    if idx==nothing
+        return "-"
+    end
+
+    # extract the value
+    line_no_keyword = strip(split(lines[idx],keyword)[end])
+    if LaMEM
+        # LaMEM output is simply everything after the keyword
+        value = line_no_keyword
+    else
+        # PETSc output usually is a table; transfer this to a Float64
+        values_vec = split(line_no_keyword," ")
+        value = values_vec[entry]
+        value = parse(Float64,value)
+    end
+
+    return value
+end
+
+
+"""
+    str = execute_command(command="ls")    
+
+Executes a command-line code and returns the output as a string
+"""
+function execute_command(ID=104810)
+
+    # execute command 
+    io = IOBuffer();
+    cmd = pipeline(`seff $ID`; stdout=io, stderr=devnull);
+
+    str = nothing
+    try 
+        run(cmd);
+        str = String(take!(io))
+        str = split(str,"\n")
+    catch
+        str = nothing
+    end
+
+    return str
+end
+
+function print_table_markdown(table; header=true)
+    nam = String.(keys(table))
+    val = values(table)
+    le = max.(length.(nam),length.(val))     # length of each fieldnames
+    sep = repeat.("-",le);
+
+    if header
+        max_n = 3;
+        names = nam;
+    else
+        max_n = 1;
+        names = val
+    end
+    str = "";
+    for i=1:max_n
+        str = str*"| ";
+        if i==2;        names = sep
+        elseif i==3;    names = val
+        end
+
+        for (j,na) in enumerate(names)
+            str *= "$(rpad(na,le[j])) | "
+        end
+        str *= "\n";
+    end
+
+    print(str)
+    return nothing
+end