diff --git a/example_apps/randomwalks.cpp b/example_apps/randomwalks.cpp index 39506809..25104890 100644 --- a/example_apps/randomwalks.cpp +++ b/example_apps/randomwalks.cpp @@ -154,7 +154,7 @@ int main(int argc, const char ** argv) { /* Detect the number of shards or preprocess an input to create them */ bool preexisting_shards; - int nshards = convert_if_notexists(filename, get_option_string("nshards", "auto"), preexisting_shards); + int nshards = convert_if_notexists(filename, get_option_string("nshards", "auto"), preexisting_shards); /* Run */ RandomWalkProgram program; diff --git a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser index d0203994..37d74d76 100644 --- a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser +++ b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser @@ -2,14 +2,17 @@ { 08FB7793FE84155DC02AAC07 /* Project object */ = { activeBuildConfigurationName = Debug; - activeExecutable = 5F648EED17B82B9900EFEE76 /* SCC */; - activeTarget = 5F648EE317B82B9900EFEE76 /* SCC */; + activeExecutable = 5FCB4690174AA11C0023D9D7 /* test_dynamicedata */; + activeTarget = 5F54B84915FD4D9F00B3842C /* test_dynamicedata */; + addToTargets = ( 5F8DD040159DDABC0034C889 /* connectedcomponents */, ); breakpoints = ( 5FCC250915A392C70003D0E9 /* streaming_pagerank.cpp:267 */, 5F648F0617B8332100EFEE76 /* stronglyconnectedcomponents.cpp:236 */, + 5F9451E417BBE83D006070F8 /* sharder.hpp:756 */, + 5F9451E917BBE8D2006070F8 /* sharder.hpp:454 */, ); codeSenseManager = 5F7A0FE21589160900748D0D /* Code sense */; executables = ( @@ -49,7 +52,7 @@ PBXFileTableDataSourceColumnSortingKey = PBXFileDataSource_Filename_ColumnID; PBXFileTableDataSourceColumnWidthsKey = ( 20, - 786, + 1049, 20, 48, 43, @@ -88,8 +91,10 @@ PBXFileDataSource_Warnings_ColumnID, ); }; - PBXPerProjectTemplateStateSaveDate = 397938024; - PBXWorkspaceStateSaveDate = 397938024; + PBXPerProjectTemplateStateSaveDate = 397677050; + PBXWorkspaceStateSaveDate = 397677050; + + }; perUserProjectItems = { 5F41F77B17B0057900C5FD90 = 5F41F77B17B0057900C5FD90 /* PBXTextBookmark */; @@ -108,6 +113,7 @@ 5F41F89817B0157600C5FD90 = 5F41F89817B0157600C5FD90 /* PBXTextBookmark */; 5F41F89917B0157600C5FD90 = 5F41F89917B0157600C5FD90 /* PBXTextBookmark */; 5F41F89A17B0157600C5FD90 = 5F41F89A17B0157600C5FD90 /* PBXTextBookmark */; + 5F41F8A117B0157600C5FD90 = 5F41F8A117B0157600C5FD90 /* PBXBookmark */; 5F41F8B317B0169E00C5FD90 = 5F41F8B317B0169E00C5FD90 /* PBXTextBookmark */; 5F41F8B417B0169E00C5FD90 = 5F41F8B417B0169E00C5FD90 /* PBXTextBookmark */; @@ -318,6 +324,8 @@ 5F6490D917BA966600EFEE76 /* PBXTextBookmark */ = 5F6490D917BA966600EFEE76 /* PBXTextBookmark */; 5F6490DA17BA966600EFEE76 /* PBXTextBookmark */ = 5F6490DA17BA966600EFEE76 /* PBXTextBookmark */; 5F6490DB17BA966600EFEE76 /* PBXTextBookmark */ = 5F6490DB17BA966600EFEE76 /* PBXTextBookmark */; + + }; sourceControlManager = 5F7A0FE11589160900748D0D /* Source Control */; userBuildSettings = { @@ -325,16 +333,16 @@ }; 5F0A1FBC16A9FBA50066FB56 /* randomwalks.cpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {964, 2431}}"; - sepNavSelRange = "{0, 0}"; - sepNavVisRange = "{0, 2191}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 2327}}"; + sepNavSelRange = "{5544, 0}"; + sepNavVisRange = "{3852, 2439}"; }; }; 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {985, 2808}}"; - sepNavSelRange = "{6410, 46}"; - sepNavVisRange = "{4600, 2034}"; + sepNavIntBoundsRect = "{{0, 0}, {985, 3003}}"; + sepNavSelRange = "{957, 0}"; + sepNavVisRange = "{2636, 2348}"; sepNavWindowFrame = "{{38, 126}, {1022, 1026}}"; }; }; @@ -400,9 +408,9 @@ }; 5F3B0889158AC5520058A8B1 /* graphchi_engine.hpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1874, 15782}}"; - sepNavSelRange = "{47743, 0}"; - sepNavVisRange = "{27025, 2953}"; + sepNavIntBoundsRect = "{{0, 0}, {1874, 15379}}"; + sepNavSelRange = "{14447, 0}"; + sepNavVisRange = "{25727, 2803}"; sepNavWindowFrame = "{{84, 84}, {1022, 1026}}"; }; }; @@ -421,7 +429,7 @@ fRef = 5F78366C15DD718A00954509 /* sharder.hpp */; name = "sharder.hpp: 658"; rLen = 0; - rLoc = 23495; + rLoc = 23493; rType = 0; vrLen = 3755; vrLoc = 21833; @@ -481,6 +489,12 @@ name = kwaymerge.hpp; path = ../src/util/kwaymerge.hpp; sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {961, 1326}}"; + sepNavSelRange = "{1198, 0}"; + sepNavVisRange = "{784, 1695}"; + sepNavWindowFrame = "{{15, 147}, {1022, 1026}}"; + }; }; 5F41F7AA17B006AE00C5FD90 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -495,7 +509,7 @@ fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; name = "conversions.hpp: 594"; rLen = 0; - rLoc = 22020; + rLoc = 22021; rType = 0; vrLen = 3496; vrLoc = 20430; @@ -523,6 +537,11 @@ name = ioutil.hpp; path = ../src/util/ioutil.hpp; sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {957, 3679}}"; + sepNavSelRange = "{5036, 0}"; + sepNavVisRange = "{4656, 913}"; + }; }; 5F41F80317B00BC600C5FD90 /* orderbydegree.hpp */ = { uiCtxt = { @@ -586,7 +605,7 @@ fRef = 5F41F7A917B006AE00C5FD90 /* kwaymerge.hpp */; name = "kwaymerge.hpp: 92"; rLen = 0; - rLoc = 2315; + rLoc = 2313; rType = 0; vrLen = 1569; vrLoc = 1050; @@ -687,7 +706,7 @@ fRef = 5FCC1D091595000F0003D0E9 /* conversions.hpp */; name = "conversions.hpp: 502"; rLen = 0; - rLoc = 18207; + rLoc = 18208; rType = 0; vrLen = 2370; vrLoc = 17609; @@ -717,16 +736,23 @@ fRef = 5F41F79E17B0068800C5FD90 /* kwaymerge.hpp */; name = "kwaymerge.hpp: 75"; rLen = 0; - rLoc = 1904; + rLoc = 1902; rType = 0; vrLen = 1455; vrLoc = 1143; }; + 5F54B84615FD4B2500B3842C /* chivector.hpp */ = { + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1227, 1742}}"; + sepNavSelRange = "{1067, 0}"; + sepNavVisRange = "{0, 1982}"; + }; + }; 5F54B84715FD4C1900B3842C /* dynamicdata_smoketest.cpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {964, 2132}}"; - sepNavSelRange = "{4552, 0}"; - sepNavVisRange = "{3421, 1821}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 2093}}"; + sepNavSelRange = "{0, 0}"; + sepNavVisRange = "{0, 2324}"; sepNavWindowFrame = "{{38, 126}, {1022, 1026}}"; }; }; @@ -863,7 +889,7 @@ hitCount = 0; ignoreCount = 0; lineNumber = 236; - modificationTime = 398029895.166603; + modificationTime = 398193925.409738; originalNumberOfMultipleMatches = 0; state = 2; }; @@ -877,11 +903,10 @@ vrLen = 1346; vrLoc = 10031; }; - 5F648F0A17B8337200EFEE76 /* XCBuildMessageTextBookmark */ = { + 5F648F0A17B8337200EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; comments = "No matching function for call to 'remove_edgev(graphchi::graphchi_edge*)'"; fRef = 5F648F0B17B8337200EFEE76 /* graph_objects.hpp */; - fallbackIsa = XCBuildMessageTextBookmark; rLen = 1; rLoc = 354; rType = 1; @@ -892,11 +917,6 @@ name = graph_objects.hpp; path = ../src/api/graph_objects.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {961, 5382}}"; - sepNavSelRange = "{3007, 194}"; - sepNavVisRange = "{2114, 2128}"; - }; }; 5F648F0C17B8337200EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -928,11 +948,10 @@ vrLen = 2281; vrLoc = 8867; }; - 5F648F2717B8340900EFEE76 /* XCBuildMessageTextBookmark */ = { + 5F648F2717B8340900EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; comments = "There are no arguments to 'num_edges' that depend on a template parameter, so a declaration of 'num_edges' must be available"; fRef = 5F648F2817B8340900EFEE76 /* graph_objects.hpp */; - fallbackIsa = XCBuildMessageTextBookmark; rLen = 1; rLoc = 358; rType = 1; @@ -943,11 +962,6 @@ name = graph_objects.hpp; path = ../src/api/graph_objects.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {961, 5421}}"; - sepNavSelRange = "{3007, 194}"; - sepNavVisRange = "{2317, 2140}"; - }; }; 5F648F2917B8340900EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -1029,11 +1043,10 @@ vrLen = 2140; vrLoc = 2317; }; - 5F648F4E17B835DD00EFEE76 /* XCBuildMessageTextBookmark */ = { + 5F648F4E17B835DD00EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; comments = "There are no arguments to 'num_edges' that depend on a template parameter, so a declaration of 'num_edges' must be available"; fRef = 5F648F4F17B835DD00EFEE76 /* graph_objects.hpp */; - fallbackIsa = XCBuildMessageTextBookmark; rLen = 1; rLoc = 358; rType = 1; @@ -1044,11 +1057,6 @@ name = graph_objects.hpp; path = ../src/api/graph_objects.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1265, 5226}}"; - sepNavSelRange = "{7734, 215}"; - sepNavVisRange = "{7054, 2286}"; - }; }; 5F648F5017B835DD00EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -1161,7 +1169,7 @@ rLen = 0; rLoc = 4552; rType = 0; - vrLen = 1821; + vrLen = 1715; vrLoc = 3421; }; 5F648F8717B83A6500EFEE76 /* PBXTextBookmark */ = { @@ -1853,15 +1861,15 @@ }; 5F64906217B96AEB00EFEE76 /* stl_function.h */ = { isa = PBXFileReference; + lastKnownFileType = sourcecode.c.h; name = stl_function.h; path = "/usr/include/c++/4.2.1/bits/stl_function.h"; sourceTree = ""; }; - 5F64906317B96AEB00EFEE76 /* XCBuildMessageTextBookmark */ = { + 5F64906317B96AEB00EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; comments = "No match for 'operator<<' in 'std::operator<< [with _Traits = std::char_traits](((std::basic_ostream >&)((std::basic_ostream >*)std::cout. std::basic_ostream<_CharT, _Traits>::operator<< [with _CharT = char, _Traits = std::char_traits]((i + 1)))), ((const char*)\". label: \")) << curlabels. std::vector<_Tp, _Alloc>::operator[] [with _Tp = labelcount_tt, _Alloc = std::allocator >](((long unsigned int)i))->labelcount_tt::label'"; fRef = 5F64906417B96AEB00EFEE76 /* labelanalysis.hpp */; - fallbackIsa = XCBuildMessageTextBookmark; rLen = 1; rLoc = 181; rType = 1; @@ -1872,11 +1880,6 @@ name = labelanalysis.hpp; path = ../src/util/labelanalysis.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1268, 2483}}"; - sepNavSelRange = "{5616, 0}"; - sepNavVisRange = "{5076, 1326}"; - }; }; 5F64906517B96AEB00EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -2015,11 +2018,10 @@ vrLen = 1316; vrLoc = 5090; }; - 5F64908017B96C1100EFEE76 /* XCBuildMessageTextBookmark */ = { + 5F64908017B96C1100EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; comments = "No matching function for call to 'std::basic_ofstream >::open(const char*&, const char [2])'"; fRef = 5F64908117B96C1100EFEE76 /* labelanalysis.hpp */; - fallbackIsa = XCBuildMessageTextBookmark; rLen = 1; rLoc = 167; rType = 1; @@ -2030,11 +2032,6 @@ name = labelanalysis.hpp; path = ../src/util/labelanalysis.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {961, 2587}}"; - sepNavSelRange = "{0, 0}"; - sepNavVisRange = "{928, 2169}"; - }; }; 5F64908217B96C1100EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -2566,6 +2563,7 @@ vrLen = 2169; vrLoc = 928; }; + 5F6490BD17B974C000EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; fRef = 5FCC1EBF1599F59A0003D0E9 /* connectedcomponents.cpp */; @@ -2596,6 +2594,7 @@ vrLen = 2986; vrLoc = 26958; }; + 5F6490C017B974C000EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; fRef = 5F648F4F17B835DD00EFEE76 /* graph_objects.hpp */; @@ -2676,6 +2675,7 @@ vrLen = 2600; vrLoc = 5683; }; + 5F6490C817BA91DE00EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; fRef = 5F64908117B96C1100EFEE76 /* labelanalysis.hpp */; @@ -2876,6 +2876,13 @@ vrLen = 1704; vrLoc = 4362; }; + 5F67654116AF31C500359562 /* vertex_data_dynamic.hpp */ = { + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1268, 2977}}"; + sepNavSelRange = "{4437, 0}"; + sepNavVisRange = "{3458, 1713}"; + }; + }; 5F74B03A15D3532600ED3EA9 /* graphlab_lda */ = { activeExec = 0; executables = ( @@ -2892,9 +2899,9 @@ }; 5F74B04F15D6CEE200ED3EA9 /* blocksplitter.cpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {893, 1196}}"; + sepNavIntBoundsRect = "{{0, 0}, {964, 1196}}"; sepNavSelRange = "{2006, 0}"; - sepNavVisRange = "{714, 2291}"; + sepNavVisRange = "{498, 2507}"; }; }; 5F74B05215D6CF4F00ED3EA9 /* blocksplitter */ = { @@ -2963,6 +2970,11 @@ name = sharder.hpp; path = ../src/preprocessing/sharder.hpp; sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1227, 13962}}"; + sepNavSelRange = "{6538, 0}"; + sepNavVisRange = "{4621, 2481}"; + }; }; 5F7A0FE11589160900748D0D /* Source Control */ = { isa = PBXSourceControlManager; @@ -3063,9 +3075,9 @@ }; 5F7A10361589266800748D0D /* vertex_data.hpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {961, 1950}}"; - sepNavSelRange = "{2013, 76}"; - sepNavVisRange = "{1377, 2283}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 1950}}"; + sepNavSelRange = "{1918, 0}"; + sepNavVisRange = "{1337, 2412}"; }; }; 5F7A103B1589266800748D0D /* stripedio.hpp */ = { @@ -3077,9 +3089,9 @@ }; 5F7A103F1589266800748D0D /* metrics.hpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {915, 4407}}"; - sepNavSelRange = "{3864, 29}"; - sepNavVisRange = "{3710, 765}"; + sepNavIntBoundsRect = "{{0, 0}, {926, 4628}}"; + sepNavSelRange = "{6728, 0}"; + sepNavVisRange = "{6377, 631}"; }; }; 5F7A10471589266800748D0D /* memoryshard.hpp */ = { @@ -3184,6 +3196,892 @@ 5FCC20ED159DE4660003D0E9 /* connectedcomponents */, ); }; + 5F9451B417BBE4CF006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F74B04F15D6CEE200ED3EA9 /* blocksplitter.cpp */; + name = "blocksplitter.cpp: 67"; + rLen = 0; + rLoc = 2006; + rType = 0; + vrLen = 2507; + vrLoc = 498; + }; + 5F9451B517BBE4CF006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 637"; + rLen = 0; + rLoc = 24279; + rType = 0; + vrLen = 4082; + vrLoc = 21897; + }; + 5F9451B617BBE4CF006070F8 /* XCBuildMessageTextBookmark */ = { + isa = PBXTextBookmark; + comments = "Expected unqualified-id before '<<' token"; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + fallbackIsa = XCBuildMessageTextBookmark; + rLen = 1; + rLoc = 657; + rType = 1; + }; + 5F9451B717BBE4CF006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 657"; + rLen = 0; + rLoc = 24119; + rType = 0; + vrLen = 3482; + vrLoc = 22625; + }; + 5F9451C117BBE51E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 657"; + rLen = 0; + rLoc = 24119; + rType = 0; + vrLen = 3482; + vrLoc = 22625; + }; + 5F9451C217BBE51E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 652"; + rLen = 0; + rLoc = 25188; + rType = 0; + vrLen = 4091; + vrLoc = 21897; + }; + 5F9451D517BBE5E9006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 720"; + rLen = 0; + rLoc = 28143; + rType = 0; + vrLen = 2033; + vrLoc = 26103; + }; + 5F9451D617BBE5E9006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 208"; + rLen = 46; + rLoc = 6492; + rType = 0; + vrLen = 1493; + vrLoc = 5153; + }; + 5F9451D717BBE5E9006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 33"; + rLen = 0; + rLoc = 956; + rType = 0; + vrLen = 1332; + vrLoc = 0; + }; + 5F9451DC17BBE5F7006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 33"; + rLen = 0; + rLoc = 956; + rType = 0; + vrLen = 1332; + vrLoc = 0; + }; + 5F9451E117BBE65D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 33"; + rLen = 0; + rLoc = 956; + rType = 0; + vrLen = 2292; + vrLoc = 4073; + }; + 5F9451E217BBE65D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 462"; + rLen = 0; + rLoc = 16688; + rType = 0; + vrLen = 2775; + vrLoc = 14252; + }; + 5F9451E417BBE83D006070F8 /* sharder.hpp:756 */ = { + isa = PBXFileBreakpoint; + actions = ( + ); + breakpointStyle = 0; + continueAfterActions = 0; + countType = 0; + delayBeforeContinue = 0; + fileReference = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + functionName = "createnextshard()"; + hitCount = 0; + ignoreCount = 0; + lineNumber = 756; + modificationTime = 398193925.409763; + originalNumberOfMultipleMatches = 1; + state = 0; + }; + 5F9451E917BBE8D2006070F8 /* sharder.hpp:454 */ = { + isa = PBXFileBreakpoint; + actions = ( + ); + breakpointStyle = 0; + continueAfterActions = 0; + countType = 0; + delayBeforeContinue = 0; + fileReference = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + functionName = "edata_flush(char * buf, char * bufptr, std::string & shard_filename, size_t totbytes)"; + hitCount = 0; + ignoreCount = 0; + lineNumber = 454; + modificationTime = 398193925.409929; + originalNumberOfMultipleMatches = 1; + state = 0; + }; + 5F9451F017BBE8F2006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 630"; + rLen = 0; + rLoc = 23553; + rType = 0; + vrLen = 2661; + vrLoc = 15893; + }; + 5F9451F117BBE8F2006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 465"; + rLen = 0; + rLoc = 16765; + rType = 0; + vrLen = 2667; + vrLoc = 15939; + }; + 5F9451F317BBE91B006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 465"; + rLen = 0; + rLoc = 16765; + rType = 0; + vrLen = 2739; + vrLoc = 15230; + }; + 5F9451FC17BBEA66006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 660"; + rLen = 0; + rLoc = 24329; + rType = 0; + vrLen = 1725; + vrLoc = 7194; + }; + 5F9451FD17BBEA66006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 660"; + rLen = 0; + rLoc = 24329; + rType = 0; + vrLen = 1725; + vrLoc = 7194; + }; + 5F94520417BBEB7D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 247"; + rLen = 0; + rLoc = 8963; + rType = 0; + vrLen = 2529; + vrLoc = 7192; + }; + 5F94520517BBEB7D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 750"; + rLen = 0; + rLoc = 29401; + rType = 0; + vrLen = 3810; + vrLoc = 27674; + }; + 5F94520D17BBECF0006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 660"; + rLen = 0; + rLoc = 24329; + rType = 0; + vrLen = 1604; + vrLoc = 7194; + }; + 5F94520E17BBECF0006070F8 /* XCBuildMessageTextBookmark */ = { + isa = PBXTextBookmark; + comments = "'intT' was not declared in this scope"; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + fallbackIsa = XCBuildMessageTextBookmark; + rLen = 1; + rLoc = 158; + rType = 1; + }; + 5F94520F17BBECF0006070F8 /* sharder.hpp */ = { + isa = PBXFileReference; + lastKnownFileType = sourcecode.cpp.h; + name = sharder.hpp; + path = ../src/preprocessing/dynamicdata/sharder.hpp; + sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1227, 11687}}"; + sepNavSelRange = "{4348, 0}"; + sepNavVisRange = "{1230, 3143}"; + }; + }; + 5F94521017BBECF0006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 162"; + rLen = 0; + rLoc = 5796; + rType = 0; + vrLen = 2037; + vrLoc = 4679; + }; + 5F94521A17BBED7A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 156"; + rLen = 0; + rLoc = 5796; + rType = 0; + vrLen = 2042; + vrLoc = 4501; + }; + 5F94521B17BBED82006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 140"; + rLen = 0; + rLoc = 5047; + rType = 0; + vrLen = 3183; + vrLoc = 2768; + }; + 5F94521C17BBED82006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */; + name = "dynamicblock.hpp: 86"; + rLen = 0; + rLoc = 2719; + rType = 0; + vrLen = 2851; + vrLoc = 1647; + }; + 5F94521D17BBED82006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F7A10361589266800748D0D /* vertex_data.hpp */; + name = "vertex_data.hpp: 76"; + rLen = 76; + rLoc = 2013; + rType = 0; + vrLen = 2412; + vrLoc = 1337; + }; + 5F94521E17BBED82006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F7A10361589266800748D0D /* vertex_data.hpp */; + name = "vertex_data.hpp: 74"; + rLen = 0; + rLoc = 1918; + rType = 0; + vrLen = 2412; + vrLoc = 1337; + }; + 5F94522217BBEDC1006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F7A10361589266800748D0D /* vertex_data.hpp */; + name = "vertex_data.hpp: 74"; + rLen = 0; + rLoc = 1918; + rType = 0; + vrLen = 2412; + vrLoc = 1337; + }; + 5F94522317BBEDC1006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F67654116AF31C500359562 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 168"; + rLen = 0; + rLoc = 5412; + rType = 0; + vrLen = 2931; + vrLoc = 2997; + }; + 5F94522417BBEDC1006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */; + rLen = 0; + rLoc = 85; + rType = 1; + }; + 5F94522517BBEDC1006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */; + name = "dynamicblock.hpp: 86"; + rLen = 0; + rLoc = 2719; + rType = 0; + vrLen = 2978; + vrLoc = 1510; + }; + 5F94522917BBEE1C006070F8 /* PBXBookmark */ = { + isa = PBXBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + }; + 5F94522A17BBEE52006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 156"; + rLen = 0; + rLoc = 5796; + rType = 0; + vrLen = 2042; + vrLoc = 4501; + }; + 5F94522E17BBEEED006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 156"; + rLen = 0; + rLoc = 5796; + rType = 0; + vrLen = 2042; + vrLoc = 4501; + }; + 5F94523117BBEF2E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 156"; + rLen = 0; + rLoc = 5796; + rType = 0; + vrLen = 2042; + vrLoc = 4501; + }; + 5F94523217BBEF2E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F67654116AF31C500359562 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 144; + rLoc = 4295; + rType = 0; + vrLen = 1713; + vrLoc = 3458; + }; + 5F94523317BBEF2E006070F8 /* XCBuildMessageTextBookmark */ = { + isa = PBXTextBookmark; + comments = "Expected `)' before '(' token"; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + fallbackIsa = XCBuildMessageTextBookmark; + rLen = 1; + rLoc = 144; + rType = 1; + }; + 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */ = { + isa = PBXFileReference; + lastKnownFileType = sourcecode.cpp.h; + name = vertex_data_dynamic.hpp; + path = ../src/engine/auxdata/dynamicdata/vertex_data_dynamic.hpp; + sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1227, 3042}}"; + sepNavSelRange = "{4437, 0}"; + sepNavVisRange = "{2279, 2864}"; + }; + }; + 5F94523517BBEF2E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F67654116AF31C500359562 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 1713; + vrLoc = 3458; + }; + 5F94523617BBEF32006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F54B84715FD4C1900B3842C /* dynamicdata_smoketest.cpp */; + name = "dynamicdata_smoketest.cpp: 1"; + rLen = 0; + rLoc = 0; + rType = 0; + vrLen = 2324; + vrLoc = 0; + }; + 5F94523717BBEF32006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2120; + vrLoc = 15; + }; + 5F94523817BBEF32006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */; + name = "dynamicblock.hpp: 86"; + rLen = 0; + rLoc = 2719; + rType = 0; + vrLen = 2978; + vrLoc = 1510; + }; + 5F94523917BBEF32006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F67654116AF31C500359562 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 140"; + rLen = 685; + rLoc = 4106; + rType = 0; + vrLen = 2694; + vrLoc = 2465; + }; + 5F94523A17BBEF32006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 2737; + vrLoc = 2147; + }; + 5F94523D17BBEF32006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2013; + vrLoc = 5163; + }; + 5F94524417BBEF66006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 1804; + vrLoc = 3445; + }; + 5F94524517BBEF68006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 2807; + vrLoc = 2270; + }; + 5F94524617BBEF68006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2013; + vrLoc = 5163; + }; + 5F94524D17BBF061006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 1698; + vrLoc = 3445; + }; + 5F94524E17BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 2786; + vrLoc = 2291; + }; + 5F94524F17BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 140"; + rLen = 0; + rLoc = 5047; + rType = 0; + vrLen = 3183; + vrLoc = 2768; + }; + 5F94525017BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F54B84615FD4B2500B3842C /* chivector.hpp */; + name = "chivector.hpp: 1"; + rLen = 0; + rLoc = 0; + rType = 0; + vrLen = 1479; + vrLoc = 1765; + }; + 5F94525117BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */; + name = "dynamicblock.hpp: 65"; + rLen = 0; + rLoc = 2165; + rType = 0; + vrLen = 2840; + vrLoc = 956; + }; + 5F94525217BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2129; + vrLoc = 6; + }; + 5F94525317BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2174; + vrLoc = 3575; + }; + 5F94525417BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2346; + vrLoc = 2636; + }; + 5F94525617BBF0A8006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 158"; + rLen = 0; + rLoc = 5032; + rType = 0; + vrLen = 2170; + vrLoc = 3575; + }; + 5F94525717BBF0A8006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2346; + vrLoc = 2636; + }; + 5F94525817BBF0AC006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 158"; + rLen = 0; + rLoc = 5032; + rType = 0; + vrLen = 2174; + vrLoc = 3575; + }; + 5F94525917BBF0AC006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2394; + vrLoc = 2636; + }; + 5F94525C17BBF0BE006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 145"; + rLen = 0; + rLoc = 4503; + rType = 0; + vrLen = 2167; + vrLoc = 3586; + }; + 5F94525D17BBF0BE006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2396; + vrLoc = 2636; + }; + 5F94526017BBF30C006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 145"; + rLen = 0; + rLoc = 4503; + rType = 0; + vrLen = 2600; + vrLoc = 426; + }; + 5F94526117BBF30C006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 2864; + vrLoc = 2279; + }; + 5F94526217BBF30C006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F54B84615FD4B2500B3842C /* chivector.hpp */; + name = "chivector.hpp: 36"; + rLen = 0; + rLoc = 1067; + rType = 0; + vrLen = 1982; + vrLoc = 0; + }; + 5F94526317BBF30C006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2396; + vrLoc = 2636; + }; + 5F94526417BBF359006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F54B84615FD4B2500B3842C /* chivector.hpp */; + name = "chivector.hpp: 36"; + rLen = 0; + rLoc = 1067; + rType = 0; + vrLen = 1982; + vrLoc = 0; + }; + 5F94526517BBF359006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F78366C15DD718A00954509 /* sharder.hpp */; + name = "sharder.hpp: 1"; + rLen = 0; + rLoc = 0; + rType = 0; + vrLen = 1797; + vrLoc = 750; + }; + 5F94526617BBF359006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 88"; + rLen = 0; + rLoc = 2570; + rType = 0; + vrLen = 2684; + vrLoc = 1211; + }; + 5F94526717BBF359006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2396; + vrLoc = 2636; + }; + 5F94526817BBF38D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 88"; + rLen = 0; + rLoc = 2570; + rType = 0; + vrLen = 2684; + vrLoc = 1211; + }; + 5F94526917BBF38D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 657"; + rLen = 0; + rLoc = 24119; + rType = 0; + vrLen = 3315; + vrLoc = 22625; + }; + 5F94526A17BBF38D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D091595000F0003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 657"; + rLen = 0; + rLoc = 24119; + rType = 0; + vrLen = 2739; + vrLoc = 6375; + }; + 5F94526B17BBF38D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2396; + vrLoc = 2636; + }; + 5F94526C17BBF45F006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 281"; + rLen = 0; + rLoc = 8950; + rType = 0; + vrLen = 3001; + vrLoc = 5981; + }; + 5F94526D17BBF45F006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FBC16A9FBA50066FB56 /* randomwalks.cpp */; + name = "randomwalks.cpp: 157"; + rLen = 0; + rLoc = 5544; + rType = 0; + vrLen = 2439; + vrLoc = 3852; + }; + 5F94526E17BBF45F006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 88"; + rLen = 0; + rLoc = 2570; + rType = 0; + vrLen = 2666; + vrLoc = 1229; + }; + 5F94526F17BBF45F006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 88"; + rLen = 0; + rLoc = 2569; + rType = 0; + vrLen = 2665; + vrLoc = 1229; + }; + 5F94527017BBF45F006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2396; + vrLoc = 2636; + }; + 5F94527717BBF508006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 122"; + rLen = 0; + rLoc = 4348; + rType = 0; + vrLen = 3143; + vrLoc = 1230; + }; + 5F94527817BBF508006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2348; + vrLoc = 2636; + }; + 5F94527A17BBF52E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 122"; + rLen = 0; + rLoc = 4348; + rType = 0; + vrLen = 3143; + vrLoc = 1230; + }; + 5F94527B17BBF52E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D041594FADE0003D0E9 /* sharder_basic.cpp */; + name = "sharder_basic.cpp: 64"; + rLen = 0; + rLoc = 2323; + rType = 0; + vrLen = 2961; + vrLoc = 13; + }; + 5F94527C17BBF52E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F78366C15DD718A00954509 /* sharder.hpp */; + name = "sharder.hpp: 207"; + rLen = 0; + rLoc = 6538; + rType = 0; + vrLen = 2481; + vrLoc = 4621; + }; + 5F94527D17BBF52E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2348; + vrLoc = 2636; + }; 5F9EBAAC1593F87D0092F7E2 /* chifilenames.hpp */ = { uiCtxt = { sepNavIntBoundsRect = "{{0, 0}, {894, 6487}}"; @@ -3218,8 +4116,23 @@ argumentStrings = ( ); autoAttachOnCrash = 1; - breakpointsEnabled = 0; + breakpointsEnabled = 1; configStateDict = { + "PBXLSLaunchAction-0" = { + PBXLSLaunchAction = 0; + PBXLSLaunchStartAction = 1; + PBXLSLaunchStdioStyle = 2; + PBXLSLaunchStyle = 0; + class = PBXLSRunLaunchConfig; + commandLineArgs = ( + ); + displayName = "Executable Runner"; + environment = { + }; + identifier = com.apple.Xcode.launch.runConfig; + remoteHostInfo = ""; + startActionInfo = ""; + }; }; customDataFormattersEnabled = 1; dataTipCustomDataFormattersEnabled = 1; @@ -3227,6 +4140,7 @@ dataTipSortType = 0; debuggerPlugin = GDBDebugging; disassemblyDisplayState = 0; + dylibVariantSuffix = ""; enableDebugStr = 1; environmentEntries = ( ); @@ -3234,9 +4148,14 @@ executableUserSymbolLevel = 0; libgmallocEnabled = 0; name = test_dynamicedata; + savedGlobals = { + }; showTypeColumn = 0; sourceDirectories = ( ); + startupPath = "/Users/akyrola/Projects/GraphCHI/graphchi-cpp/"; + variableFormatDictionary = { + }; }; 5FCB4691174AA11C0023D9D7 /* gensgd */ = { isa = PBXExecutable; @@ -3302,17 +4221,17 @@ }; 5FCC1D041594FADE0003D0E9 /* sharder_basic.cpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1153, 1105}}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 1105}}"; sepNavSelRange = "{2323, 0}"; - sepNavVisRange = "{74, 2804}"; + sepNavVisRange = "{13, 2961}"; sepNavWindowFrame = "{{61, 155}, {999, 976}}"; }; }; 5FCC1D091595000F0003D0E9 /* conversions.hpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {985, 8268}}"; - sepNavSelRange = "{18199, 0}"; - sepNavVisRange = "{17609, 2370}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 10374}}"; + sepNavSelRange = "{24118, 0}"; + sepNavVisRange = "{6375, 2739}"; }; }; 5FCC1D0E159501720003D0E9 /* sharder_basic */ = { @@ -3354,6 +4273,11 @@ name = conversions.hpp; path = ../src/preprocessing/conversions.hpp; sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1227, 10361}}"; + sepNavSelRange = "{8950, 0}"; + sepNavVisRange = "{5981, 3001}"; + }; }; 5FCC1DD71599194B0003D0E9 /* graphchi_basic_includes.hpp */ = { uiCtxt = { @@ -3643,11 +4567,6 @@ name = graph_objects.hpp; path = ../src/api/graph_objects.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1268, 5213}}"; - sepNavSelRange = "{3809, 188}"; - sepNavVisRange = "{2761, 1480}"; - }; }; 5FCC229215A1EC330003D0E9 /* als_graphlab.cpp */ = { uiCtxt = { @@ -3808,10 +4727,27 @@ hitCount = 0; ignoreCount = 0; lineNumber = 267; - modificationTime = 398029895.166577; + + modificationTime = 398193925.409677; originalNumberOfMultipleMatches = 1; state = 0; }; + 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */ = { + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1227, 1833}}"; + sepNavSelRange = "{2165, 0}"; + sepNavVisRange = "{956, 2840}"; + }; + }; + 5FF7749917B3E1A200F84939 /* sharder.hpp */ = { + uiCtxt = { + + + sepNavIntBoundsRect = "{{0, 0}, {1268, 11362}}"; + sepNavSelRange = "{5614, 0}"; + sepNavVisRange = "{4679, 2037}"; + }; + }; 5FF774DD17B3E94100F84939 /* minimumspanningforest.cpp */ = { uiCtxt = { sepNavIntBoundsRect = "{{0, 0}, {961, 5694}}"; diff --git a/graphchi_xcode/graphchi_xcode.xcodeproj/project.pbxproj b/graphchi_xcode/graphchi_xcode.xcodeproj/project.pbxproj index 5c28de98..3db3fbff 100644 --- a/graphchi_xcode/graphchi_xcode.xcodeproj/project.pbxproj +++ b/graphchi_xcode/graphchi_xcode.xcodeproj/project.pbxproj @@ -426,6 +426,7 @@ 5FCC242B15A378DC0003D0E9 /* streaming_pagerank.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = streaming_pagerank.cpp; sourceTree = ""; }; 5FCC243415A378E70003D0E9 /* streaming_pagerank */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = streaming_pagerank; sourceTree = BUILT_PRODUCTS_DIR; }; 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = dynamicblock.hpp; sourceTree = ""; }; + 5FF7749917B3E1A200F84939 /* sharder.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = sharder.hpp; sourceTree = ""; }; 5FF774DD17B3E94100F84939 /* minimumspanningforest.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = minimumspanningforest.cpp; sourceTree = ""; }; /* End PBXFileReference section */ @@ -910,6 +911,7 @@ 5FCC1CD81594F1890003D0E9 /* sharder.hpp */, 5FCC1D041594FADE0003D0E9 /* sharder_basic.cpp */, 5F74B04F15D6CEE200ED3EA9 /* blocksplitter.cpp */, + 5FF7749817B3E1A200F84939 /* dynamicdata */, 5F41F80217B00BC600C5FD90 /* util */, ); path = preprocessing; @@ -1182,6 +1184,14 @@ path = graphlab2_1_GAS_api; sourceTree = ""; }; + 5FF7749817B3E1A200F84939 /* dynamicdata */ = { + isa = PBXGroup; + children = ( + 5FF7749917B3E1A200F84939 /* sharder.hpp */, + ); + path = dynamicdata; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ diff --git a/src/api/dynamicdata/chivector.hpp b/src/api/dynamicdata/chivector.hpp index 517d6dbc..af28bf27 100644 --- a/src/api/dynamicdata/chivector.hpp +++ b/src/api/dynamicdata/chivector.hpp @@ -32,38 +32,32 @@ #include #include +#include "graphchi_types.hpp" namespace graphchi { #define MINCAPACITY 2 - -/** - * Pool the extension parts of chi-vectors - */ -template -class extension_pool { - -}; - -template +template class chivector { uint16_t nsize; uint16_t ncapacity; + HeaderType hdr; // statically sized header T * data; std::vector * extensions; // TODO: use a more memory efficient system? public: typedef T element_type_t; + typedef HeaderType header_t; typedef uint32_t sizeword_t; chivector() { extensions = NULL; } - chivector(uint16_t sz, uint16_t cap, T * dataptr) : data(dataptr) { + chivector(uint16_t sz, uint16_t cap, HeaderType hdr, T * dataptr) : hdr(hdr), data(dataptr) { nsize = sz; ncapacity = cap; assert(cap >= nsize); @@ -77,6 +71,18 @@ class chivector { } } + HeaderType & header() { + return hdr; + } + + const HeaderType & header() const { + return hdr; + } + + double* const& get_data() const { + return data; + } + void write(T * dest) { int sz = (int) this->size(); for(int i=0; i < sz; i++) { @@ -131,6 +137,8 @@ class chivector { void clear() { nsize = 0; + if(extensions != NULL) + extensions -> clear(); } // TODO: iterators diff --git a/src/engine/auxdata/dynamicdata/vertex_data_dynamic.hpp b/src/engine/auxdata/dynamicdata/vertex_data_dynamic.hpp index 573f1b70..21d33050 100644 --- a/src/engine/auxdata/dynamicdata/vertex_data_dynamic.hpp +++ b/src/engine/auxdata/dynamicdata/vertex_data_dynamic.hpp @@ -142,9 +142,9 @@ namespace graphchi { std::string bfilename = blockfilename(blockid); if (!file_exists(bfilename)) { mkdir(dirname.c_str(), 0777); - size_t initsize = verticesperblock * sizeof(typename VertexDataType::sizeword_t); + size_t initsize = verticesperblock * (sizeof(typename VertexDataType::header_t) + sizeof(typename VertexDataType::sizeword_t)); int f = open(bfilename.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); - uint8_t * zeros = (uint8_t *) calloc(verticesperblock, sizeof(typename VertexDataType::sizeword_t)); + uint8_t * zeros = (uint8_t *) calloc(verticesperblock, sizeof(typename VertexDataType::header_t) + sizeof(typename VertexDataType::sizeword_t)); write_compressed(f, zeros, initsize); free(zeros); @@ -162,9 +162,10 @@ namespace graphchi { db.fd = iomgr->open_session(blockfname, false, true); int realsize = get_block_uncompressed_size(blockfname, -1); assert(realsize > 0); - iomgr->managed_malloc(db.fd, &db.data, realsize, 0); iomgr->managed_preada_now(db.fd, &db.data, realsize, 0); + assert(db.data); + db.dblock = new dynamicdata_block(verticesperblock, (uint8_t *)db.data, realsize); return db; } diff --git a/src/graphchi_types.hpp b/src/graphchi_types.hpp index e32a447f..f5587c62 100644 --- a/src/graphchi_types.hpp +++ b/src/graphchi_types.hpp @@ -52,6 +52,11 @@ namespace graphchi { } } }; + + // Empty data type + struct dummy { + }; + } diff --git a/src/preprocessing/conversions.hpp b/src/preprocessing/conversions.hpp index c8b7766d..af53cbc6 100644 --- a/src/preprocessing/conversions.hpp +++ b/src/preprocessing/conversions.hpp @@ -1,4 +1,4 @@ -;/** +/** * @file * @author Aapo Kyrola * @version 1.0 @@ -54,10 +54,7 @@ #endif namespace graphchi { - - struct dummy { - }; - + template struct dummyC { operator T() { return T(); } @@ -207,6 +204,7 @@ namespace graphchi { * Converts graph from an edge list format. Input may contain * value for the edges. Self-edges are ignored. */ + template void convert_edgelist(std::string inputfile, sharder &sharderobj, bool multivalue_edges=false) { @@ -275,7 +273,7 @@ namespace graphchi { logstream(LOG_FATAL) << "Each edge needs at least one value." << std::endl; assert(vals.size() > 0); } - sharderobj.preprocessing_add_edge_multival(from, to, vals); + sharderobj.preprocessing_add_edge_multival(from, to, dummy(), vals); } #else @@ -391,8 +389,8 @@ namespace graphchi { std::string line; // current line // handle header line - int n = 0; // number of nodes - int m = 0; // number of edges + int n=0; // number of nodes + int m=0; // number of edges int weighted; // indicates weight scheme: if (std::getline(graphFile, line)) { @@ -657,6 +655,7 @@ namespace graphchi { * Converts a graph input to shards with no edge values. Preprocessing has several steps, * see sharder.hpp for more information. */ + static int VARIABLE_IS_NOT_USED convert_none(std::string basefilename, std::string nshards_string); static int VARIABLE_IS_NOT_USED convert_none(std::string basefilename, std::string nshards_string) { sharder sharderobj(basefilename); diff --git a/src/preprocessing/dynamicdata/sharder.hpp b/src/preprocessing/dynamicdata/sharder.hpp new file mode 100644 index 00000000..2a8ea1f2 --- /dev/null +++ b/src/preprocessing/dynamicdata/sharder.hpp @@ -0,0 +1,932 @@ +/** + * @file + * @author Aapo Kyrola + * @version 1.0 + * + * @section LICENSE + * + * Copyright [2012] [Aapo Kyrola, Guy Blelloch, Carlos Guestrin / Carnegie Mellon University] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + + * + * @section DESCRIPTION + * + * Sharder converts a graph into shards which the GraphChi engine + * can process. For DYNAMICEDATA + */ + + +#ifndef DYNAMICEDATA + error(Can be used only with DYNAMICEDATA) +#endif + +#ifndef GRAPHCHI_SHARDER_DEF +#define GRAPHCHI_SHARDER_DEF + + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "api/chifilenames.hpp" +#include "api/graphchi_context.hpp" +#include "graphchi_types.hpp" +#include "io/stripedio.hpp" +#include "logger/logger.hpp" +#include "engine/auxdata/degree_data.hpp" +#include "metrics/metrics.hpp" +#include "metrics/reps/basic_reporter.hpp" +#include "shards/memoryshard.hpp" +#include "shards/slidingshard.hpp" +#include "output/output.hpp" +#include "util/ioutil.hpp" +#include "util/kwaymerge.hpp" +#include "util/qsort.hpp" +namespace graphchi { + + +#define SHARDER_BUFSIZE (64 * 1024 * 1024) + + enum ProcPhase { COMPUTE_INTERVALS=1, SHOVEL=2 }; + + + template + struct edge_with_value { + vid_t src; + vid_t dst; + HeaderDataType hdr; + std::vector value; + + // For dynamic edge data, we need to know if the value needs to be added + // to the vector, or are we storing an empty vector. + bool is_chivec_value; + edge_with_value() : value(std::vector(0)) {} + + edge_with_value(vid_t src, vid_t dst, std::vector value) : src(src), dst(dst), value(value) { + } + + edge_with_value(vid_t src, vid_t dst, std::vector value, HeaderDataType hdr) : src(src), dst(dst), hdr(hdr), value(value){ + } + + inline void docopy(const edge_with_value &x) { + + src = x.src; + dst = x.dst; + hdr = x.hdr; + is_chivec_value = x.is_chivec_value; + + + if (&x == this) { + // This can happen! (Also fixed an issue in binary_minhep that could cause this). + assert(false); + return; + } + + // Copy values one by one + size_t n = x.value.size(); + + if (n > 0) { + value.reserve(n); + + for(size_t i=0; i &x) : value(0) { + docopy(x); + } + + edge_with_value(edge_with_value &x) : value(0) { + docopy(x); + } + + + + edge_with_value& operator= (const edge_with_value &x) { + this->value = std::vector(0); + this->docopy(x); + return *this; + } + + edge_with_value& operator= (edge_with_value &x) { + this->value = std::vector(0); + this->docopy(x); + return *this; + } + + + // Order primarily by dst, then by src + bool operator< (edge_with_value &x2) { + return (dst < x2.dst); + } + + // TODO: use buffered I/O + void reade(int f) { + read(f, &src, sizeof(vid_t)); + read(f, &dst, sizeof(vid_t)); + read(f, &is_chivec_value, sizeof(bool)); + + if (is_chivec_value) { + uint16_t nvalues; + read(f, &nvalues, sizeof(uint16_t)); + value.resize(nvalues); + read(f, &value[0], sizeof(VectorElementType) * nvalues); + read(f, &hdr, sizeof(HeaderDataType)); + } + //std::cout << src << " - " << dst << " " << *((int*)&value[0]) << std::endl; + } + + void writee(int f) { + writea(f, &src, sizeof(vid_t)); + writea(f, &dst, sizeof(vid_t)); + writea(f, &is_chivec_value, sizeof(bool)); + + uint16_t nvalues = value.size(); + assert(value.size() < 1<<16); + writea(f, &nvalues, sizeof(uint16_t)); + writea(f, &value[0], sizeof(VectorElementType) * nvalues); + writea(f, &hdr, sizeof(HeaderDataType)); + } + + + bool stopper() { return src == 0 && dst == 0; } + }; + + template + bool edge_t_src_less(const edge_with_value &a, const edge_with_value &b) { + if (a.src == b.src) { + return a.dst < b.dst; + } + return a.src < b.src; + } + + template + bool edge_t_dst_less(const edge_with_value &a, const edge_with_value &b) { + return a.dst < b.dst; + } + + + + template + struct shard_flushinfo { + std::string shovelname; + size_t numedges; + edge_with_value * buffer; + vid_t max_vertex; + + shard_flushinfo(std::string shovelname, vid_t max_vertex, size_t numedges, edge_with_value * buffer) : + shovelname(shovelname), numedges(numedges), buffer(buffer), max_vertex(max_vertex) {} + + void flush() { + /* Sort */ + // TODO: remove duplicates here! + logstream(LOG_INFO) << "Sorting shovel: " << shovelname << ", max:" << max_vertex << std::endl; + quickSort(buffer, numedges, edge_t_dst_less); + logstream(LOG_INFO) << "Sort done." << shovelname << std::endl; + int f = open(shovelname.c_str(), O_WRONLY | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); + //writea(f, buffer, numedges * sizeof(edge_with_value)); + + writea(f, &numedges, sizeof(size_t)); + + /* Write the edges into file. This is quite a bit more complicated than with non-dynamic data... */ + for(size_t i=0; i &edge = buffer[i]; + edge.writee(f); + } + + close(f); + delete [] buffer; + } + }; + + // Run in a thread + template + static void * shard_flush_run(void * _info) { + shard_flushinfo * task = (shard_flushinfo*)_info; + task->flush(); + return NULL; + } + + + template + struct shovel_merge_source : public merge_source > { + + size_t bufsize_bytes; + std::string shovelfile; + size_t idx; + size_t bufidx; + std::vector > buffer; + int f; + size_t edgesread; + size_t numedges; + size_t bufsize_edges; + + shovel_merge_source(size_t bufsize_bytes, std::string shovelfile) : bufsize_bytes(bufsize_bytes), + shovelfile(shovelfile), idx(0), bufidx(0) { + f = open(shovelfile.c_str(), O_RDONLY); + edgesread = 0; + if (f < 0) { + logstream(LOG_ERROR) << "Could not open shovel file: " << shovelfile << std::endl; + printf("Error: %d, %s\n", errno, strerror(errno)); + } + + assert(f>=0); + + bufsize_edges = bufsize_bytes / sizeof(edge_with_value); + read(f, &numedges, sizeof(size_t)); + + std::cout << "numedges: " << numedges << std::endl; + load_next(); + } + + virtual ~shovel_merge_source() {} + + void finish() { + close(f); + remove(shovelfile.c_str()); + } + + void load_next() { + size_t nread = 0; + buffer.clear(); + while(nread < bufsize_bytes && edgesread < numedges) { + edge_with_value edge; + edge.reade(f); + buffer.push_back(edge); + nread += sizeof(vid_t) * 2 + sizeof(uint16_t) + sizeof(HeaderDataType) + edge.value.size() * sizeof(VectorElementType); + edgesread++; + } + bufidx = 0; + } + + bool has_more() { + return idx < numedges; + } + + edge_with_value next() { + if (bufidx == bufsize_edges) { + load_next(); + } + idx++; + if (idx == numedges) { + edge_with_value &x = buffer[bufidx++]; + finish(); + return x; + } + return buffer[bufidx++]; + } + }; + + template + class sharder : public merge_sink > { + + typedef edge_with_value edge_t; + + protected: + std::string basefilename; + + vid_t max_vertex_id; + + /* Sharding */ + int nshards; + std::vector< std::pair > intervals; + + + int vertexchunk; + size_t nedges; + std::string prefix; + + int compressed_block_size; + + int * bufptrs; + size_t edgedatasize; + size_t ebuffer_size; + size_t edges_per_block; + + vid_t filter_max_vertex; + + + bool no_edgevalues; + edge_t last_added_edge; + + metrics m; + + + size_t curshovel_idx; + size_t shovelsize; + int numshovels; + size_t shoveled_edges; + bool shovel_sorted; + edge_with_value * curshovel_buffer; + std::vector shovelthreads; + + public: + + sharder(std::string basefilename) : basefilename(basefilename), m("sharder") { + + edgedatasize = sizeof(VectorElementType); + no_edgevalues = false; + compressed_block_size = 4096 * 1024; + filter_max_vertex = 0; + curshovel_buffer = NULL; + while (compressed_block_size % sizeof(VectorElementType) != 0) compressed_block_size++; + edges_per_block = compressed_block_size / sizeof(VectorElementType); + } + + + virtual ~sharder() { + if (curshovel_buffer == NULL) delete [] curshovel_buffer; + } + + + void set_max_vertex_id(vid_t maxid) { + filter_max_vertex = maxid; + } + + void set_no_edgevalues() { + no_edgevalues = true; + } + + /** + * Call to start a preprocessing session. + */ + void start_preprocessing() { + m.start_time("preprocessing"); + numshovels = 0; + shovelsize = (1024l * 1024l * size_t(get_option_int("membudget_mb", 1024)) / 4l / sizeof(edge_with_value)); + curshovel_idx = 0; + + logstream(LOG_INFO) << "Starting preprocessing, shovel size: " << shovelsize << std::endl; + + curshovel_buffer = new edge_with_value[shovelsize]; + + assert(curshovel_buffer != NULL); + + shovelthreads.clear(); + + /* Write the maximum vertex id place holder - to be filled later */ + max_vertex_id = 0; + shoveled_edges = 0; + } + + /** + * Call to finish the preprocessing session. + */ + void end_preprocessing() { + m.stop_time("preprocessing"); + flush_shovel(false); + } + + void flush_shovel(bool async=true) { + /* Flush in separate thread unless the last one */ + shard_flushinfo * flushinfo = new shard_flushinfo(shovel_filename(numshovels), max_vertex_id, curshovel_idx, curshovel_buffer); + + if (!async) { + curshovel_buffer = NULL; + flushinfo->flush(); + + /* Wait for threads to finish */ + logstream(LOG_INFO) << "Waiting shoveling threads..." << std::endl; + for(int i=0; i < (int)shovelthreads.size(); i++) { + pthread_join(shovelthreads[i], NULL); + } + } else { + if (shovelthreads.size() > 2) { + logstream(LOG_INFO) << "Too many outstanding shoveling threads..." << std::endl; + + for(int i=0; i < (int)shovelthreads.size(); i++) { + pthread_join(shovelthreads[i], NULL); + } + shovelthreads.clear(); + } + curshovel_buffer = new edge_with_value[shovelsize]; + pthread_t t; + int ret = pthread_create(&t, NULL, shard_flush_run, (void*)flushinfo); + shovelthreads.push_back(t); + assert(ret>=0); + } + numshovels++; + curshovel_idx=0; + } + + /** + * Add edge to be preprocessed with a value. + */ + void preprocessing_add_edge(vid_t from, vid_t to, VectorElementType val, bool input_value=false) { + if (from == to) { + // Do not allow self-edges + return; + } + assert(!input_value); + edge_with_value e(from, to, std::vector(0)); + + e.is_chivec_value = input_value; + last_added_edge = e; + curshovel_buffer[curshovel_idx++] = e; + if (curshovel_idx == shovelsize) { + flush_shovel(); + } + + max_vertex_id = std::max(std::max(from, to), max_vertex_id); + shoveled_edges++; + } + + void preprocessing_add_edge_multival(vid_t from, vid_t to, HeaderDataType hdr, std::vector & vals) { + if (from == to) { + // Do not allow self-edges + return; + } + edge_with_value e(from, to, vals, hdr); + e.is_chivec_value = true; + + curshovel_buffer[curshovel_idx++] = e; + if (curshovel_idx == shovelsize) { + flush_shovel(); + } + + max_vertex_id = std::max(std::max(from, to), max_vertex_id); + shoveled_edges++; + } + + + /** + * Add edge without value to be preprocessed + */ + void preprocessing_add_edge(vid_t from, vid_t to) { + preprocessing_add_edge(from, to, VectorElementType()); + } + + /** Buffered write function */ + template + void bwrite(int f, char * buf, char * &bufptr, T val) { + if (bufptr + sizeof(T) - buf >= SHARDER_BUFSIZE) { + writea(f, buf, bufptr - buf); + bufptr = buf; + } + *((T*)bufptr) = val; + bufptr += sizeof(T); + } + + int blockid; + + void edata_flush(char * buf, char * bufptr, std::string & shard_filename, size_t totbytes) { + int len = (int) (bufptr - buf); + + m.start_time("edata_flush"); + + std::cout << shard_filename << std::endl; + + std::string block_filename = filename_shard_edata_block(shard_filename, blockid, compressed_block_size); + int f = open(block_filename.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); + write_compressed(f, buf, len); + close(f); + + m.stop_time("edata_flush"); + + + // Write block's uncompressed size + write_block_uncompressed_size(block_filename, len); + blockid++; + } + + template + void bwrite_edata(char * &buf, char * &bufptr, T val, size_t & totbytes, std::string & shard_filename, + size_t & edgecounter) { + if (no_edgevalues) return; + + if (edgecounter == edges_per_block) { + edata_flush(buf, bufptr, shard_filename, totbytes); + bufptr = buf; + edgecounter = 0; + } + + // Check if buffer is big enough + if (bufptr - buf + sizeof(T) > ebuffer_size) { + ebuffer_size *= 2; + logstream(LOG_DEBUG) << "Increased buffer size to: " << ebuffer_size << std::endl; + size_t ptroff = bufptr - buf; // Remember the offset + buf = (char *) realloc(buf, ebuffer_size); + bufptr = buf + ptroff; + } + + totbytes += sizeof(T); + *((T*)bufptr) = val; + bufptr += sizeof(T); + } + + + /** + * Executes sharding. + * @param nshards_string the number of shards as a number, or "auto" for automatic determination + */ + int execute_sharding(std::string nshards_string) { + m.start_time("execute_sharding"); + + determine_number_of_shards(nshards_string); + write_shards(); + + m.stop_time("execute_sharding"); + + /* Print metrics */ + basic_reporter basicrep; + m.report(basicrep); + + return nshards; + } + + /** + * Sharding. This code might be hard to read - modify very carefully! + */ + protected: + + virtual void determine_number_of_shards(std::string nshards_string) { + if (nshards_string.find("auto") != std::string::npos || nshards_string == "0") { + logstream(LOG_INFO) << "Determining number of shards automatically." << std::endl; + + int membudget_mb = get_option_int("membudget_mb", 1024); + logstream(LOG_INFO) << "Assuming available memory is " << membudget_mb << " megabytes. " << std::endl; + logstream(LOG_INFO) << " (This can be defined with configuration parameter 'membudget_mb')" << std::endl; + + size_t numedges = shoveled_edges; + + double max_shardsize = membudget_mb * 1024. * 1024. / 8; + logstream(LOG_INFO) << "Determining maximum shard size: " << (max_shardsize / 1024. / 1024.) << " MB." << std::endl; + + // For dynamic edge data, more working memory is needed, thus the number of shards is larger. + nshards = (int) ( 2 + 4 * (numedges * sizeof(VectorElementType) / max_shardsize) + 0.5); + + } else { + nshards = atoi(nshards_string.c_str()); + } + assert(nshards > 0); + logstream(LOG_INFO) << "Number of shards to be created: " << nshards << std::endl; + } + + + protected: + + void one_shard_intervals() { + assert(nshards == 1); + std::string fname = filename_intervals(basefilename, nshards); + FILE * f = fopen(fname.c_str(), "w"); + intervals.push_back(std::pair(0, max_vertex_id)); + fprintf(f, "%u\n", max_vertex_id); + fclose(f); + + /* Write meta-file with the number of vertices */ + std::string numv_filename = basefilename + ".numvertices"; + f = fopen(numv_filename.c_str(), "w"); + fprintf(f, "%u\n", 1 + max_vertex_id); + fclose(f); + + assert(nshards == (int)intervals.size()); + } + + + std::string shovel_filename(int idx) { + std::stringstream ss; + ss << basefilename << sizeof(VectorElementType) << "." << idx << ".shovel"; + return ss.str(); + } + + + int lastpart; + degree * degrees; + + virtual void finish_shard(int shard, edge_t * shovelbuf, size_t shovelsize) { + m.start_time("shard_final"); + blockid = 0; + size_t edgecounter = 0; + + logstream(LOG_INFO) << "Starting final processing for shard: " << shard << std::endl; + + std::string fname = filename_shard_adj(basefilename, shard, nshards); + std::string edfname = filename_shard_edata(basefilename, shard, nshards); + std::string edblockdirname = dirname_shard_edata_block(edfname, compressed_block_size); + + /* Make the block directory */ + if (!no_edgevalues) + mkdir(edblockdirname.c_str(), 0777); + size_t numedges = shovelsize / sizeof(edge_t); + + logstream(LOG_DEBUG) << "Shovel size:" << shovelsize << " edges: " << numedges << std::endl; + + m.start_time("finish_shard.sort"); + std::cout << "Going to sort: " << shovelbuf[0].value.size() << std::endl; + quickSort(shovelbuf, (int)numedges, edge_t_src_less); + m.stop_time("finish_shard.sort"); + + // Create the final file + int f = open(fname.c_str(), O_WRONLY | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); + if (f < 0) { + logstream(LOG_ERROR) << "Could not open " << fname << " error: " << strerror(errno) << std::endl; + } + assert(f >= 0); + int trerr = ftruncate(f, 0); + assert(trerr == 0); + + char * buf = (char*) malloc(SHARDER_BUFSIZE); + char * bufptr = buf; + + char * ebuf = (char*) malloc(compressed_block_size); + ebuffer_size = compressed_block_size; + char * ebufptr = ebuf; + + vid_t curvid=0; + vid_t lastdst = 0xffffffff; + size_t num_uniq_edges = 0; + size_t last_edge_count = 0; + size_t istart = 0; + size_t tot_edatabytes = 0; + for(size_t i=0; i <= numedges; i++) { + if (i % 10000000 == 0) logstream(LOG_DEBUG) << i << " / " << numedges << std::endl; + edge_t edge = (i < numedges ? shovelbuf[i] : edge_t(0, 0, std::vector())); // Last "element" is a stopper + + if (lastdst == edge.dst && edge.src == curvid) { + // Currently not supported + logstream(LOG_ERROR) << "Duplicate edge in the stream - aborting" << std::endl; + assert(false); + } + lastdst = edge.dst; + + if (!edge.stopper()) { + assert(i < numedges); + /* If we have dynamic edge data, we need to write the header of chivector - if there are edge values */ + bwrite_edata(ebuf, ebufptr, edge.hdr, tot_edatabytes, edfname, edgecounter); + + if (edge.is_chivec_value) { + // Need to check how many values for this edge + + int count = edge.value.size(); + + assert(count < 32768); + typename chivector::sizeword_t szw; + ((uint16_t *) &szw)[0] = (uint16_t)count; // Sizeword with length and capacity = count + ((uint16_t *) &szw)[1] = (uint16_t)count; + bwrite_edata::sizeword_t>(ebuf, ebufptr, szw, tot_edatabytes, edfname, edgecounter); + for(int j=0; j < count; j++) { + bwrite_edata(ebuf, ebufptr, edge.value[j], tot_edatabytes, edfname, edgecounter); + } + + + } else { + // Just write size word with zero + bwrite_edata::sizeword_t >(ebuf, ebufptr, 0, tot_edatabytes, edfname, edgecounter); + } + num_uniq_edges++; + edgecounter++; // Increment edge counter here --- notice that dynamic edata case makes two or more calls to bwrite_edata before incrementing + } + if (degrees != NULL && edge.src != edge.dst) { + degrees[edge.src].outdegree++; + degrees[edge.dst].indegree++; + } + + if ((edge.src != curvid) || edge.stopper()) { + // New vertex + + size_t count = num_uniq_edges - 1 - last_edge_count; + last_edge_count = num_uniq_edges - 1; + if (edge.stopper()) count++; + assert(count>0 || curvid==0); + if (count>0) { + if (count < 255) { + uint8_t x = (uint8_t)count; + bwrite(f, buf, bufptr, x); + } else { + bwrite(f, buf, bufptr, 0xff); + bwrite(f, buf, bufptr, (uint32_t)count); + } + } + + for(size_t j=istart; j < i; j++) { + bwrite(f, buf, bufptr, shovelbuf[j].dst); + } + istart = i; + + // Handle zeros + if (!edge.stopper()) { + if (edge.src - curvid > 1 || (i == 0 && edge.src>0)) { + int nz = edge.src - curvid - 1; + if (i == 0 && edge.src > 0) nz = edge.src; // border case with the first one + do { + bwrite(f, buf, bufptr, 0); + nz--; + int tnz = std::min(254, nz); + bwrite(f, buf, bufptr, (uint8_t) tnz); + nz -= tnz; + } while (nz>0); + } + } + curvid = edge.src; + } + } + + /* Flush buffers and free memory */ + writea(f, buf, bufptr - buf); + free(buf); + delete [] shovelbuf; + close(f); + + /* Write edata size file */ + if (!no_edgevalues) { + edata_flush(ebuf, ebufptr, edfname, tot_edatabytes); + + std::string sizefilename = edfname + ".size"; + std::ofstream ofs(sizefilename.c_str()); + + ofs << num_uniq_edges * sizeof(int); // For dynamic edge data, write the number of edges. + ofs.close(); + } + free(ebuf); + + m.stop_time("shard_final"); + } + + /* Begin: Kway -merge sink interface */ + + size_t edges_per_shard; + size_t cur_shard_counter; + size_t shard_capacity; + size_t sharded_edges; + int shardnum; + edge_with_value * sinkbuffer; + vid_t prevvid; + vid_t this_interval_start; + + virtual void add(edge_with_value val) { + if (cur_shard_counter >= edges_per_shard && val.dst != prevvid) { + createnextshard(); + } + + if (cur_shard_counter == shard_capacity) { + /* Really should have a way to limit shard sizes, but probably not needed in practice */ + logstream(LOG_WARNING) << "Shard " << shardnum << " overflowing! " << cur_shard_counter << " / " << shard_capacity << std::endl; + shard_capacity = (size_t) (1.2 * shard_capacity); + sinkbuffer = (edge_with_value*) realloc(sinkbuffer, shard_capacity * sizeof(edge_with_value)); + } + + std::cout << cur_shard_counter << ". add: " << val.src << ", " << val.dst << ", " << val.value.size() << std::endl; + assert(val.value.size() < 10000000); + sinkbuffer[cur_shard_counter++] = val; + std::cout << cur_shard_counter << ". add: " << sinkbuffer[cur_shard_counter-1].src + << ", " << sinkbuffer[cur_shard_counter-1].dst << ", " << sinkbuffer[cur_shard_counter-1].value.size() << std::endl; + + prevvid = val.dst; + sharded_edges++; + + + } + + void createnextshard() { + assert(shardnum < nshards); + intervals.push_back(std::pair(this_interval_start, (shardnum == nshards - 1 ? max_vertex_id : prevvid))); + this_interval_start = prevvid + 1; + finish_shard(shardnum++, sinkbuffer, cur_shard_counter * sizeof(edge_with_value)); + sinkbuffer = new edge_with_value[shard_capacity]; + //(edge_with_value *) malloc(shard_capacity * sizeof(edge_with_value)); + cur_shard_counter = 0; + + std::cout << "Allocated sinkbuffer: " << shard_capacity * sizeof(edge_with_value) << ", shard_capacity=" << shard_capacity << std::endl; + std::cout << "sinkbuffer:" << sinkbuffer << std::endl; + + // Adjust edges per hard so that it takes into account how many edges have been spilled now + logstream(LOG_INFO) << "Remaining edges: " << (shoveled_edges - sharded_edges) << " remaining shards:" << (nshards - shardnum) + << " edges per shard=" << edges_per_shard << std::endl; + if (shardnum < nshards) edges_per_shard = (shoveled_edges - sharded_edges) / (nshards - shardnum); + logstream(LOG_INFO) << "Edges per shard: " << edges_per_shard << std::endl; + + } + + virtual void done() { + createnextshard(); + if (shoveled_edges != sharded_edges) { + logstream(LOG_INFO) << "Shoveled " << shoveled_edges << " but sharded " << sharded_edges << " edges" << std::endl; + } + assert(shoveled_edges == sharded_edges); + + logstream(LOG_INFO) << "Created " << shardnum << " shards, expected: " << nshards << std::endl; + assert(shardnum <= nshards); + delete [] sinkbuffer; + sinkbuffer = NULL; + + /* Write intervals */ + std::string fname = filename_intervals(basefilename, nshards); + FILE * f = fopen(fname.c_str(), "w"); + + if (f == NULL) { + logstream(LOG_ERROR) << "Could not open file: " << fname << " error: " << + strerror(errno) << std::endl; + } + assert(f != NULL); + for(int i=0; i<(int)intervals.size(); i++) { + fprintf(f, "%u\n", intervals[i].second); + } + fclose(f); + + /* Write meta-file with the number of vertices */ + std::string numv_filename = basefilename + ".numvertices"; + f = fopen(numv_filename.c_str(), "w"); + fprintf(f, "%u\n", 1 + max_vertex_id); + fclose(f); + } + + /* End: Kway -merge sink interface */ + + + + /** + * Write the shard by sorting the shovel file and compressing the + * adjacency information. + * To support different shard types, override this function! + */ + virtual void write_shards() { + + size_t membudget_mb = (size_t) get_option_int("membudget_mb", 1024); + + // Check if we have enough memory to keep track + // of the vertex degrees in-memory (heuristic) + degrees = (degree *) calloc(1 + max_vertex_id, sizeof(degree)); + + // KWAY MERGE + sharded_edges = 0; + edges_per_shard = shoveled_edges / nshards + 1; + shard_capacity = edges_per_shard / 2 * 3; // Shard can go 50% over + shardnum = 0; + this_interval_start = 0; + sinkbuffer = new edge_with_value [shard_capacity]; + logstream(LOG_INFO) << "Edges per shard: " << edges_per_shard << " nshards=" << nshards << " total: " << shoveled_edges << std::endl; + cur_shard_counter = 0; + + /* Initialize kway merge sources */ + size_t B = membudget_mb * 1024 * 1024 / 2 / numshovels; + while (B % sizeof(edge_with_value) != 0) B++; + logstream(LOG_INFO) << "Buffer size in merge phase: " << B << std::endl; + prevvid = (-1); + std::vector< merge_source > *> sources; + for(int i=0; i < numshovels; i++) { + sources.push_back(new shovel_merge_source(B, shovel_filename(i))); + } + + kway_merge > merger(sources, this); + merger.merge(); + + // Delete sources + for(int i=0; i < (int)sources.size(); i++) { + delete (shovel_merge_source *)sources[i]; + } + + + + std::string degreefname = filename_degree_data(basefilename); + int degreeOutF = open(degreefname.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); + if (degreeOutF < 0) { + logstream(LOG_ERROR) << "Could not create: " << degreeOutF << std::endl; + assert(degreeOutF >= 0); + } + + writea(degreeOutF, degrees, sizeof(degree) * (1 + max_vertex_id)); + free(degrees); + close(degreeOutF); + } + + + typedef char dummy_t; + + typedef sliding_shard slidingshard_t; + typedef memory_shard memshard_t; + + + }; // End class sharder + + + +}; // namespace + + +#endif + + + diff --git a/src/preprocessing/sharder.hpp b/src/preprocessing/sharder.hpp index 15f10019..240f5882 100644 --- a/src/preprocessing/sharder.hpp +++ b/src/preprocessing/sharder.hpp @@ -26,11 +26,10 @@ * can process. */ -/** - * @section TODO - * Change all C-style IO to Unix-style IO. - */ +#ifdef DYNAMICEDATA +#include "preprocessing/dynamicdata/sharder.hpp" // Special sharder if dynamic edata used. +#else #ifndef GRAPHCHI_SHARDER_DEF #define GRAPHCHI_SHARDER_DEF @@ -62,9 +61,8 @@ #include "util/ioutil.hpp" #include "util/radixSort.hpp" #include "util/kwaymerge.hpp" -#ifdef DYNAMICEDATA -#include "util/qsort.hpp" -#endif + + namespace graphchi { template class sharded_graph_output; @@ -86,20 +84,10 @@ namespace graphchi { vid_t dst; EdgeDataType value; -#ifdef DYNAMICEDATA - // For dynamic edge data, we need to know if the value needs to be added - // to the vector, or are we storing an empty vector. - bool is_chivec_value; - uint16_t valindex; -#endif + edge_with_value() {} - edge_with_value(vid_t src, vid_t dst, EdgeDataType value) : src(src), dst(dst), value(value) { -#ifdef DYNAMICEDATA - is_chivec_value = false; - valindex = 0; -#endif - } + edge_with_value(vid_t src, vid_t dst, EdgeDataType value) : src(src), dst(dst), value(value) {} // Order primarily by dst, then by src bool operator< (edge_with_value &x2) { @@ -113,11 +101,6 @@ namespace graphchi { template bool edge_t_src_less(const edge_with_value &a, const edge_with_value &b) { if (a.src == b.src) { -#ifdef DYNAMICEDATA - if (a.dst == b.dst) { - return a.valindex < b.valindex; - } -#endif return a.dst < b.dst; } return a.src < b.src; @@ -268,10 +251,6 @@ namespace graphchi { DuplicateEdgeFilter * duplicate_edge_filter; bool no_edgevalues; -#ifdef DYNAMICEDATA - edge_t last_added_edge; -#endif - metrics m; @@ -385,13 +364,7 @@ namespace graphchi { return; } edge_with_value e(from, to, val); -#ifdef DYNAMICEDATA - e.is_chivec_value = input_value; - if (e.src == last_added_edge.src && e.dst == last_added_edge.dst) { - e.valindex = last_added_edge.valindex + 1; - } - last_added_edge = e; -#endif + curshovel_buffer[curshovel_idx++] = e; if (curshovel_idx == shovelsize) { flush_shovel(); @@ -401,16 +374,10 @@ namespace graphchi { shoveled_edges++; } -#ifdef DYNAMICEDATA - void preprocessing_add_edge_multival(vid_t from, vid_t to, std::vector & vals) { - typename std::vector::iterator iter; - for(iter=vals.begin(); iter != vals.end(); ++iter) { - preprocessing_add_edge(from, to, *iter, true); - } - max_vertex_id = std::max(std::max(from, to), max_vertex_id); - } + -#endif + + /** * Add edge without value to be preprocessed @@ -444,14 +411,6 @@ namespace graphchi { close(f); m.stop_time("edata_flush"); - - -#ifdef DYNAMICEDATA - // Write block's uncompressed size - write_block_uncompressed_size(block_filename, len); - -#endif - blockid++; } @@ -519,12 +478,7 @@ namespace graphchi { logstream(LOG_INFO) << "Determining maximum shard size: " << (max_shardsize / 1024. / 1024.) << " MB." << std::endl; nshards = (int) ( 1 + (numedges * sizeof(FinalEdgeDataType) / max_shardsize) + 0.5); - -#ifdef DYNAMICEDATA - // For dynamic edge data, more working memory is needed, thus the number of shards is larger. - nshards = (int) ( 2 + 4 * (numedges * sizeof(FinalEdgeDataType) / max_shardsize) + 0.5); -#endif - + } else { nshards = atoi(nshards_string.c_str()); } @@ -586,11 +540,8 @@ namespace graphchi { logstream(LOG_DEBUG) << "Shovel size:" << shovelsize << " edges: " << numedges << std::endl; m.start_time("finish_shard.sort"); -#ifndef DYNAMICEDATA iSort(shovelbuf, (int)numedges, max_vertex_id, srcF()); -#else - quickSort(shovelbuf, (int)numedges, edge_t_src_less); -#endif + m.stop_time("finish_shard.sort"); // Remove duplicates @@ -634,59 +585,18 @@ namespace graphchi { char * ebufptr = ebuf; vid_t curvid=0; -#ifdef DYNAMICEDATA - vid_t lastdst = 0xffffffff; - int jumpover = 0; - size_t num_uniq_edges = 0; - size_t last_edge_count = 0; -#endif + size_t istart = 0; size_t tot_edatabytes = 0; for(size_t i=0; i <= numedges; i++) { if (i % 10000000 == 0) logstream(LOG_DEBUG) << i << " / " << numedges << std::endl; -#ifdef DYNAMICEDATA - i += jumpover; // With dynamic values, there might be several values for one edge, and thus the edge repeated in the data. - jumpover = 0; -#endif //DYNAMICEDATA + edge_t edge = (i < numedges ? shovelbuf[i] : edge_t(0, 0, EdgeDataType())); // Last "element" is a stopper -#ifdef DYNAMICEDATA - - if (lastdst == edge.dst && edge.src == curvid) { - // Currently not supported - logstream(LOG_ERROR) << "Duplicate edge in the stream - aborting" << std::endl; - assert(false); - } - lastdst = edge.dst; -#endif + if (!edge.stopper()) { -#ifndef DYNAMICEDATA bwrite_edata(ebuf, ebufptr, FinalEdgeDataType(edge.value), tot_edatabytes, edfname, edgecounter); -#else - /* If we have dynamic edge data, we need to write the header of chivector - if there are edge values */ - if (edge.is_chivec_value) { - // Need to check how many values for this edge - int count = 1; - while(shovelbuf[i + count].valindex == count) { count++; } - - assert(count < 32768); - - typename chivector::sizeword_t szw; - ((uint16_t *) &szw)[0] = (uint16_t)count; // Sizeword with length and capacity = count - ((uint16_t *) &szw)[1] = (uint16_t)count; - bwrite_edata::sizeword_t>(ebuf, ebufptr, szw, tot_edatabytes, edfname, edgecounter); - for(int j=0; j < count; j++) { - bwrite_edata(ebuf, ebufptr, EdgeDataType(shovelbuf[i + j].value), tot_edatabytes, edfname, edgecounter); - } - jumpover = count - 1; // Jump over - } else { - // Just write size word with zero - bwrite_edata(ebuf, ebufptr, 0, tot_edatabytes, edfname, edgecounter); - } - num_uniq_edges++; - -#endif edgecounter++; // Increment edge counter here --- notice that dynamic edata case makes two or more calls to bwrite_edata before incrementing } if (degrees != NULL && edge.src != edge.dst) { @@ -696,13 +606,8 @@ namespace graphchi { if ((edge.src != curvid) || edge.stopper()) { // New vertex -#ifndef DYNAMICEDATA size_t count = i - istart; -#else - size_t count = num_uniq_edges - 1 - last_edge_count; - last_edge_count = num_uniq_edges - 1; - if (edge.stopper()) count++; -#endif + assert(count>0 || curvid==0); if (count>0) { if (count < 255) { @@ -714,23 +619,10 @@ namespace graphchi { } } -#ifndef DYNAMICEDATA for(size_t j=istart; j < i; j++) { bwrite(f, buf, bufptr, shovelbuf[j].dst); } -#else - // Special dealing with dynamic edata because some edges can be present multiple - // times in the shovel. - for(size_t j=istart; j < i; j++) { - if (j == istart || shovelbuf[j - 1].dst != shovelbuf[j].dst) { - bwrite(f, buf, bufptr, shovelbuf[j].dst); - } - } -#endif istart = i; -#ifdef DYNAMICEDATA - istart += jumpover; -#endif // Handle zeros if (!edge.stopper()) { @@ -762,12 +654,7 @@ namespace graphchi { std::string sizefilename = edfname + ".size"; std::ofstream ofs(sizefilename.c_str()); -#ifndef DYNAMICEDATA - ofs << tot_edatabytes; -#else - ofs << num_uniq_edges * sizeof(int); // For dynamic edge data, write the number of edges. -#endif - + ofs << tot_edatabytes; ofs.close(); } free(ebuf); @@ -869,16 +756,7 @@ namespace graphchi { // of the vertex degrees in-memory (heuristic) bool count_degrees_inmem = membudget_mb * 1024 * 1024 / 3 > max_vertex_id * sizeof(degree); degrees = NULL; -#ifdef DYNAMICEDATA - if (!count_degrees_inmem) { - /* Temporary: force in-memory count of degrees because the PSW-based computation - is not yet compatible with dynamic edge data. - */ - logstream(LOG_WARNING) << "Dynamic edge data support only sharding when the vertex degrees can be computed in-memory." << std::endl; - logstream(LOG_WARNING) << "If the program gets very slow (starts swapping), the data size is too big." << std::endl; - count_degrees_inmem = true; - } -#endif + if (count_degrees_inmem) { degrees = (degree *) calloc(1 + max_vertex_id, sizeof(degree)); } @@ -913,11 +791,8 @@ namespace graphchi { if (!count_degrees_inmem) { -#ifndef DYNAMICEDATA // Use memory-efficient (but slower) method to create degree-data - create_degree_file(); -#endif - + create_degree_file(); } else { std::string degreefname = filename_degree_data(basefilename); int degreeOutF = open(degreefname.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); @@ -940,7 +815,6 @@ namespace graphchi { typedef memory_shard memshard_t; -#ifndef DYNAMICEDATA void create_degree_file() { // Initialize IO stripedio * iomgr = new stripedio(m); @@ -1061,7 +935,6 @@ namespace graphchi { m.stop_time("degrees.runtime"); delete iomgr; } -#endif template friend class sharded_graph_output; }; // End class sharder @@ -1141,5 +1014,6 @@ namespace graphchi { #endif +#endif // ifdef DYNAMICEDATA diff --git a/src/shards/dynamicdata/dynamicblock.hpp b/src/shards/dynamicdata/dynamicblock.hpp index 503d1eb0..4b77d22b 100644 --- a/src/shards/dynamicdata/dynamicblock.hpp +++ b/src/shards/dynamicdata/dynamicblock.hpp @@ -83,9 +83,11 @@ namespace graphchi { uint8_t * ptr = data; for(int i=0; i < nitems; i++) { assert(ptr - data <= datasize); + typename ET::header_t hdr = *((typename ET::header_t *) ptr); + ptr += sizeof(typename ET::header_t); typename ET::sizeword_t * sz = ((typename ET::sizeword_t *) ptr); ptr += sizeof(typename ET::sizeword_t); - chivecs[i] = ET(((uint16_t *)sz)[0], ((uint16_t *)sz)[1], (typename ET::element_type_t *) ptr); + chivecs[i] = ET(((uint16_t *)sz)[0], ((uint16_t *)sz)[1], hdr, (typename ET::element_type_t *) ptr); ptr += (int) ((uint16_t *)sz)[1] * sizeof(typename ET::element_type_t); } } @@ -100,17 +102,24 @@ namespace graphchi { // First compute size size = 0; for(int i=0; i < nitems; i++) { - size += chivecs[i].capacity() * sizeof(typename ET::element_type_t) + sizeof(typename ET::sizeword_t); + size += chivecs[i].capacity() * sizeof(typename ET::element_type_t) + sizeof(typename ET::sizeword_t) + sizeof(typename ET::header_t); } *outdata = (uint8_t *) malloc(size); uint8_t * ptr = *outdata; for(int i=0; i < nitems; i++) { ET & vec = chivecs[i]; + + /* Write header */ + ((typename ET::header_t *) ptr)[0] = vec.header(); + ptr += sizeof(typename ET::header_t); + + /* Write size information */ ((uint16_t *) ptr)[0] = vec.size(); ((uint16_t *) ptr)[1] = vec.capacity(); - ptr += sizeof(typename ET::sizeword_t); + + /* Write elements */ vec.write((typename ET::element_type_t *) ptr); ptr += vec.capacity() * sizeof(typename ET::element_type_t); } diff --git a/src/tests/test_dynamicedata_loader.cpp b/src/tests/test_dynamicedata_loader.cpp index f9bd8450..c9a15f73 100644 --- a/src/tests/test_dynamicedata_loader.cpp +++ b/src/tests/test_dynamicedata_loader.cpp @@ -30,6 +30,7 @@ #define DYNAMICEDATA 1 #define DYNAMICVERTEXDATA 1 +#define GRAPHCHI_DISABLE_COMPRESSION 1 #include @@ -42,8 +43,15 @@ using namespace graphchi; * Type definitions. Remember to create suitable graph shards using the * Sharder-program. */ -typedef chivector VertexDataType; -typedef chivector EdgeDataType; +struct header_t { + vid_t a; + bool b; + header_t() {} + header_t(vid_t a, bool b): a(a), b(b) {} +}; + +typedef chivector VertexDataType; +typedef chivector EdgeDataType; size_t checksum = 0; size_t shouldbe = 0; @@ -60,7 +68,7 @@ struct DynamicDataLoaderTestProgram : public GraphChiProgram &vertex, graphchi_context &gcontext) { for(int i=0; i < vertex.num_edges(); i++) { - chivector * evector = vertex.edge(i)->get_vector(); + chivector * evector = vertex.edge(i)->get_vector(); assert(evector != NULL); @@ -77,13 +85,16 @@ struct DynamicDataLoaderTestProgram : public GraphChiProgramget(k) == expected); } + assert(evector->header().a >= vertex.id() + vertex.edge(i)->vertex_id() + gcontext.iteration); + assert(evector->header().b == (vertex.id() + vertex.edge(i)->vertex_id()) % 2); + evector->header().a = vertex.id() + vertex.edge(i)->vertex_id() + gcontext.iteration; lock.lock(); checksum += evector->get(0); lock.unlock(); } // Modify vertex data by adding values there */ - chivector * vvector = vertex.get_vector(); + chivector * vvector = vertex.get_vector(); int numitems = vertex.id() % 10; for(int i=0; iadd(vertex.id() * 982192l + i); // Arbitrary @@ -124,35 +135,43 @@ struct DynamicDataLoaderTestProgram : public GraphChiProgram sharderobj(filename); + sharderobj.start_preprocessing(); + std::cout << "Generating data..." << std::endl; - const char * fname = filename.c_str(); - FILE * f = fopen(fname, "w"); - set_conf("filetype", "edgelist"); + shouldbe = 0; - int totalVertices = 200000; // 2 million + int totalVertices = 200000; for(int i=0; i < totalVertices; i++) { - int nedges = random() % 50; + int nedges = random() % 10; for(int j=0; j < nedges; j++) { int dst = (totalVertices / nedges) * j + i % nedges; if (dst != i) { + std::vector edgevec; if ((i + dst) % 3 == 1) { - fprintf(f, "%d\t%d\t%d:%d:%d\n", i, dst, i + dst, i + dst + 1, i + dst + 2); + edgevec.push_back(i + dst); + edgevec.push_back(i + dst + 1); + edgevec.push_back(i + dst + 2); } else { - fprintf(f, "%d\t%d\t%d\n", i, dst, i + dst); - + edgevec.push_back(i + dst); } - shouldbe += 2 * (i + dst); + shouldbe += 2 * (i + dst); + + sharderobj.preprocessing_add_edge_multival(i, dst, header_t(i + dst, (i + dst) % 2), edgevec); } } - } - fclose(f); + } + sharderobj.end_preprocessing(); + + return sharderobj.execute_sharding(get_option_string("nshards", "auto")); + } -class VertexValidator : public VCallback > { +class VertexValidator : public VCallback { public: - virtual void callback(vid_t vertex_id, chivector &vec) { + virtual void callback(vid_t vertex_id, VertexDataType &vec) { int numitems = vertex_id % 10; assert(vec.size() == numitems); @@ -184,9 +203,8 @@ int main(int argc, const char ** argv) { bool scheduler = false; // Whether to use selective scheduling /* Generate data */ - generatedata(filename); - set_conf("filetype", "multivalueedgelist"); - int nshards = convert_if_notexists(filename, "3"); + int nshards = generatedata(filename); + checksum = 0; /* Run */ diff --git a/src/util/binary_minheap.hpp b/src/util/binary_minheap.hpp index 3126254a..710759bb 100644 --- a/src/util/binary_minheap.hpp +++ b/src/util/binary_minheap.hpp @@ -82,7 +82,9 @@ class binary_minheap { T min() { return values[0]; } void extractMin() { - values[0] = values[sz - 1]; + if (sz > 1) { + values[0] = values[sz - 1]; + } decrHeapSize(); minHeapify(0); }