Skip to content

Commit

Permalink
add IEnumerable iterator to sheets, add row caching back, better thre…
Browse files Browse the repository at this point in the history 
…ading
  • Loading branch information
@NotAdam
NotAdam committed Jun 15, 2020
1 parent b3876be commit f699e96
Show file tree
Hide file tree
Showing 7 changed files with 135 additions and 125 deletions.
16 changes: 9 additions & 7 deletions src/Lumina.Example/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Lumina.Data;
using Lumina.Excel;
using Lumina.Excel.GeneratedSheets;
Expand Down Expand Up @@ -60,10 +61,11 @@ public override void SaveFile( string path )
static void Main( string[] args )
{
var lumina = new Lumina( args[ 0 ] );


bool exitThread = false;
var handleThread = new Thread( () =>
{
while( true )
while( !exitThread )
{
lumina.ProcessFileHandleQueue();
Thread.Yield();
Expand All @@ -74,23 +76,23 @@ static void Main( string[] args )

// excel reading
var actionTimeline = lumina.GetExcelSheet< ActionTimeline >();
var atRows = actionTimeline.GetRows().Take( 5 );
var atRows = actionTimeline.Take( 5 );

foreach( var row in atRows )
{
Console.WriteLine( $"name: {row.Key}" );
}

var zoneSharedGroup = lumina.GetExcelSheet< ZoneSharedGroup >();
var zsgRows = zoneSharedGroup.GetRows().Take( 5 );
var zsgRows = zoneSharedGroup.Take( 5 );

foreach( var row in zsgRows )
{
Console.WriteLine( $"ZoneSharedGroup({row.RowId}.{row.SubRowId}) quest1: {row.Quest1}" );
}

// dump conditions
foreach( var condition in lumina.GetExcelSheet< Condition >().GetRows() )
foreach( var condition in lumina.GetExcelSheet< Condition >() )
{
Console.WriteLine( $"condition {condition.RowId:000}: {condition.LogMessage.Value?.Text}" );
}
Expand All @@ -103,8 +105,8 @@ static void Main( string[] args )
var aetheryte = file.ExdMap.First( m => m.Key == "Aetheryte" );

Console.WriteLine( $"aetheryte: id: {aetheryte.Value} name: {aetheryte.Key}" );
handleThread.Abort();

exitThread = true;
}
}
}
9 changes: 7 additions & 2 deletions src/Lumina/Data/FileHandleManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public FileHandle< T > CreateHandle< T >( string path ) where T : FileResource
/// </summary>
public void ProcessQueue()
{
while( _fileQueue.Any() )
while( HasPendingFileLoads )
{
var res = _fileQueue.TryDequeue( out var weakRef );
if( res && weakRef.TryGetTarget( out var handle ) )
Expand All @@ -46,6 +46,11 @@ public void ProcessQueue()
}
}
}


/// <summary>
/// Whether the file queue contains any files that are yet to be loaded
/// </summary>
public bool HasPendingFileLoads => !_fileQueue.IsEmpty;

}
}
30 changes: 21 additions & 9 deletions src/Lumina/Data/SqPack.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,9 @@ public bool ShouldConvertEndianness
!BitConverter.IsLittleEndian && SqPackHeader.platformId != PlatformId.PS3;
}

protected ConcurrentDictionary< long, WeakReference< FileResource > > _fileCache;
protected readonly Dictionary< long, WeakReference< FileResource > > FileCache;

protected readonly object CacheLock = new object();

internal SqPack( FileInfo file, Lumina lumina )
{
Expand All @@ -63,7 +65,7 @@ internal SqPack( FileInfo file, Lumina lumina )
_Lumina = lumina;

// always init the cache just in case the should cache setting is changed later
_fileCache = new ConcurrentDictionary< long, WeakReference< FileResource > >();
FileCache = new Dictionary< long, WeakReference< FileResource > >();

File = file;

Expand All @@ -77,7 +79,7 @@ internal SqPack( FileInfo file, Lumina lumina )

protected T GetCachedFile< T >( long offset ) where T : FileResource
{
if( !_fileCache.TryGetValue( offset, out var weakRef ) )
if( !FileCache.TryGetValue( offset, out var weakRef ) )
{
return null;
}
Expand Down Expand Up @@ -108,16 +110,31 @@ public SqPackFileInfo GetFileMetadata( long offset )

public T ReadFile< T >( long offset ) where T : FileResource
{
if( _Lumina.Options.CacheFileResources )
if( !_Lumina.Options.CacheFileResources )
{
return ReadFileInternal< T >( offset );
}


lock( CacheLock )
{
var obj = GetCachedFile< T >( offset );

if( obj != null )
{
return obj;
}

var file = ReadFileInternal< T >( offset );

FileCache[ offset ] = new WeakReference< FileResource >( file );

return file;
}
}

protected T ReadFileInternal< T >( long offset ) where T : FileResource
{
using var fs = File.OpenRead();
using var br = new BinaryReader( fs );
using var ms = new MemoryStream();
Expand Down Expand Up @@ -187,11 +204,6 @@ public T ReadFile< T >( long offset ) where T : FileResource
file.Reader = new BinaryReader( file.FileStream );
file.FileStream.Position = 0;

if( _Lumina.Options.CacheFileResources )
{
_fileCache[ offset ] = new WeakReference< FileResource >( file );
}

file.LoadFile();

return file;
Expand Down
49 changes: 30 additions & 19 deletions src/Lumina/Excel/ExcelModule.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ namespace Lumina.Excel
{
public class ExcelModule
{
private readonly Lumina _Lumina;
private readonly Lumina _lumina;

/// <summary>
/// Mapping between internal IDs used to index sheets loaded at startup to their name.
Expand All @@ -28,18 +28,20 @@ public class ExcelModule
/// </summary>
public readonly List< string > SheetNames;

private readonly ConcurrentDictionary< Tuple< Language, string >, ExcelSheetImpl > _sheetCache;
private readonly Dictionary< Tuple< Language, string >, ExcelSheetImpl > _sheetCache;

private readonly object _sheetCreateLock = new object();

public ExcelModule( Lumina lumina )
{
_Lumina = lumina;
_lumina = lumina;
ImmutableIdToSheetMap = new Dictionary< int, string >();
SheetNames = new List< string >();

_sheetCache = new ConcurrentDictionary< Tuple< Language, string >, ExcelSheetImpl >();
_sheetCache = new Dictionary< Tuple< Language, string >, ExcelSheetImpl >();

// load all sheet names first
var files = _Lumina.GetFile< ExcelListFile >( "exd/root.exl" );
var files = _lumina.GetFile< ExcelListFile >( "exd/root.exl" );

Debug.WriteLine( $"got {files.ExdMap.Count} exlt entries" );

Expand Down Expand Up @@ -76,7 +78,7 @@ public string BuildExcelHeaderPath( string name )
/// <returns>An <see cref="ExcelSheet{T}"/> if the sheet exists, null if it does not</returns>
public ExcelSheet< T > GetSheet< T >() where T : IExcelRow
{
return GetSheet< T >( _Lumina.Options.DefaultExcelLanguage );
return GetSheet< T >( _lumina.Options.DefaultExcelLanguage );
}

/// <summary>
Expand All @@ -97,7 +99,10 @@ public ExcelSheet< T > GetSheet< T >( Language language ) where T : IExcelRow
return null;
}

return GetSheet< T >( attr.Name, language, attr.ColumnHash );
lock( _sheetCreateLock )
{
return GetSheet< T >( attr.Name, language, attr.ColumnHash );
}
}

/// <summary>
Expand All @@ -110,7 +115,10 @@ public ExcelSheet< T > GetSheet< T >( Language language ) where T : IExcelRow
/// <returns>An <see cref="ExcelSheet{T}"/> if the sheet exists, null if it does not</returns>
public ExcelSheet< T > GetSheet< T >( string name ) where T : IExcelRow
{
return GetSheet< T >( name, _Lumina.Options.DefaultExcelLanguage, null );
lock( _sheetCreateLock )
{
return GetSheet< T >( name, _lumina.Options.DefaultExcelLanguage, null );
}
}

private ExcelSheet< T > GetSheet< T >( string name, Language language, uint? expectedHash ) where T : IExcelRow
Expand All @@ -128,20 +136,23 @@ private ExcelSheet< T > GetSheet< T >( string name, Language language, uint? exp

if( _sheetCache.TryGetValue( id, out sheet ) )
{
return sheet as ExcelSheet< T >;
if( sheet is ExcelSheet< T > checkedSheet )
{
return checkedSheet;
}
}

if( !SheetNames.Contains( name ) )
// create new sheet
var path = BuildExcelHeaderPath( name );
var headerFile = _lumina.GetFile< ExcelHeaderFile >( path );

if( headerFile == null )
{
return null;
}

// create new sheet
var path = BuildExcelHeaderPath( name );
var headerFile = _Lumina.GetFile< ExcelHeaderFile >( path );

// validate checksum if enabled and we have a hash that we expect to find
if( _Lumina.Options.PanicOnSheetChecksumMismatch && expectedHash.HasValue )
if( _lumina.Options.PanicOnSheetChecksumMismatch && expectedHash.HasValue )
{
var actualHash = headerFile.GetColumnsHash();
if( actualHash != expectedHash )
Expand All @@ -150,7 +161,7 @@ private ExcelSheet< T > GetSheet< T >( string name, Language language, uint? exp
}
}

var newSheet = (ExcelSheet< T >)Activator.CreateInstance( typeof( ExcelSheet< T > ), headerFile, name, language, _Lumina );
var newSheet = (ExcelSheet< T >)Activator.CreateInstance( typeof( ExcelSheet< T > ), headerFile, name, language, _lumina );
newSheet.GenerateFilePages();

// kinda a shit hack but basically this enforces a single language for a sheet that has no localisation
Expand All @@ -163,7 +174,7 @@ private ExcelSheet< T > GetSheet< T >( string name, Language language, uint? exp
id = idNoLanguage;
}

_sheetCache.TryAdd( id, newSheet );
_sheetCache[ id ] = newSheet;

return newSheet;
}
Expand All @@ -185,9 +196,9 @@ public ExcelSheetImpl GetSheetRaw( string name, Language language = Language.Non

// create new sheet
var path = BuildExcelHeaderPath( name );
var headerFile = _Lumina.GetFile< ExcelHeaderFile >( path );
var headerFile = _lumina.GetFile< ExcelHeaderFile >( path );

var newSheet = (ExcelSheetImpl)Activator.CreateInstance( typeof( ExcelSheetImpl ), headerFile, name, language, _Lumina );
var newSheet = (ExcelSheetImpl)Activator.CreateInstance( typeof( ExcelSheetImpl ), headerFile, name, language, _lumina );
newSheet.GenerateFilePages();

return newSheet;
Expand Down
Loading

0 comments on commit f699e96

Please sign in to comment.