From f8fe11432f176bb7178205cf04b33df38cc4697e Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Thu, 8 Aug 2013 10:30:49 -0400 Subject: [PATCH 01/14] removed dynamic edge data ifdefs and instead have now separate sharder for dynamic data --- .../graphchi_xcode.xcodeproj/akyrola.pbxuser | 219 +--- .../graphchi_xcode.xcodeproj/project.pbxproj | 10 + src/preprocessing/conversions.hpp | 4 +- src/preprocessing/dynamicdata/sharder.hpp | 1145 +++++++++++++++++ src/preprocessing/sharder.hpp | 167 +-- 5 files changed, 1202 insertions(+), 343 deletions(-) create mode 100644 src/preprocessing/dynamicdata/sharder.hpp diff --git a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser index 7289b0f3..f9e1aa5b 100644 --- a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser +++ b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser @@ -5,7 +5,6 @@ activeExecutable = 5FCC2131159DF1F20003D0E9 /* trianglecounting */; activeTarget = 5FCC2128159DF1F20003D0E9 /* trianglecounting */; addToTargets = ( - 5FCC2128159DF1F20003D0E9 /* trianglecounting */, ); breakpoints = ( 5FCC250915A392C70003D0E9 /* streaming_pagerank.cpp:267 */, @@ -86,37 +85,21 @@ PBXFileDataSource_Warnings_ColumnID, ); }; - PBXPerProjectTemplateStateSaveDate = 397416203; - PBXWorkspaceStateSaveDate = 397416203; + PBXPerProjectTemplateStateSaveDate = 397514523; + PBXWorkspaceStateSaveDate = 397514523; }; perUserProjectItems = { 5F41F77B17B0057900C5FD90 = 5F41F77B17B0057900C5FD90 /* PBXTextBookmark */; 5F41F77C17B0057900C5FD90 = 5F41F77C17B0057900C5FD90 /* PBXTextBookmark */; - 5F41F7AA17B006AE00C5FD90 = 5F41F7AA17B006AE00C5FD90 /* PBXTextBookmark */; 5F41F7B417B0073000C5FD90 = 5F41F7B417B0073000C5FD90 /* PBXTextBookmark */; 5F41F7CD17B0085B00C5FD90 = 5F41F7CD17B0085B00C5FD90 /* PBXTextBookmark */; - 5F41F80517B00D0700C5FD90 = 5F41F80517B00D0700C5FD90 /* PBXBookmark */; 5F41F80617B00D1700C5FD90 = 5F41F80617B00D1700C5FD90 /* PBXTextBookmark */; 5F41F80717B00D1700C5FD90 = 5F41F80717B00D1700C5FD90 /* PBXTextBookmark */; - 5F41F80817B00D1700C5FD90 = 5F41F80817B00D1700C5FD90 /* PBXTextBookmark */; - 5F41F82D17B00FED00C5FD90 = 5F41F82D17B00FED00C5FD90 /* PBXBookmark */; - 5F41F83317B00FED00C5FD90 = 5F41F83317B00FED00C5FD90 /* PBXBookmark */; - 5F41F85017B010A400C5FD90 = 5F41F85017B010A400C5FD90 /* PBXBookmark */; - 5F41F87017B012BB00C5FD90 = 5F41F87017B012BB00C5FD90 /* PBXBookmark */; 5F41F89817B0157600C5FD90 = 5F41F89817B0157600C5FD90 /* PBXTextBookmark */; 5F41F89917B0157600C5FD90 = 5F41F89917B0157600C5FD90 /* PBXTextBookmark */; 5F41F89A17B0157600C5FD90 = 5F41F89A17B0157600C5FD90 /* PBXTextBookmark */; - 5F41F8A117B0157600C5FD90 = 5F41F8A117B0157600C5FD90 /* PBXBookmark */; - 5F41F8B317B0169E00C5FD90 = 5F41F8B317B0169E00C5FD90 /* PBXTextBookmark */; - 5F41F8B417B0169E00C5FD90 = 5F41F8B417B0169E00C5FD90 /* PBXTextBookmark */; - 5F41F8BE17B0170500C5FD90 = 5F41F8BE17B0170500C5FD90 /* PBXTextBookmark */; - 5F41F8BF17B0170500C5FD90 = 5F41F8BF17B0170500C5FD90 /* PBXTextBookmark */; - 5F41F8C017B0170500C5FD90 = 5F41F8C017B0170500C5FD90 /* PBXTextBookmark */; - 5F41F8C117B0170500C5FD90 = 5F41F8C117B0170500C5FD90 /* PBXTextBookmark */; - 5F41F8C217B0170500C5FD90 = 5F41F8C217B0170500C5FD90 /* PBXTextBookmark */; - 5F41F8C317B0170500C5FD90 = 5F41F8C317B0170500C5FD90 /* PBXTextBookmark */; - 5F41F8C417B0170500C5FD90 = 5F41F8C417B0170500C5FD90 /* PBXTextBookmark */; - 5F41F8C517B0170500C5FD90 = 5F41F8C517B0170500C5FD90 /* PBXTextBookmark */; + 5FF7741517B170E300F84939 = 5FF7741517B170E300F84939 /* PBXBookmark */; + 5FF7741617B170E300F84939 = 5FF7741617B170E300F84939 /* PBXTextBookmark */; }; sourceControlManager = 5F7A0FE11589160900748D0D /* Source Control */; userBuildSettings = { @@ -213,19 +196,6 @@ name = kwaymerge.hpp; path = ../src/util/kwaymerge.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {950, 1521}}"; - sepNavSelRange = "{1904, 0}"; - sepNavVisRange = "{1143, 1384}"; - }; - }; - 5F41F7AA17B006AE00C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - comments = "Implicit conversion shortens 64-bit value into a 32-bit value"; - fRef = 5F41F7A917B006AE00C5FD90 /* kwaymerge.hpp */; - rLen = 1; - rLoc = 74; - rType = 1; }; 5F41F7B417B0073000C5FD90 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -268,10 +238,6 @@ sepNavVisRange = "{2765, 2025}"; }; }; - 5F41F80517B00D0700C5FD90 /* PBXBookmark */ = { - isa = PBXBookmark; - fRef = 5FCC2104159DEC0E0003D0E9 /* trianglecounting.cpp */; - }; 5F41F80617B00D1700C5FD90 /* PBXTextBookmark */ = { isa = PBXTextBookmark; fRef = 5F41F7F717B00ADE00C5FD90 /* ioutil.hpp */; @@ -292,32 +258,6 @@ vrLen = 2291; vrLoc = 714; }; - 5F41F80817B00D1700C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5FCC242B15A378DC0003D0E9 /* streaming_pagerank.cpp */; - name = "streaming_pagerank.cpp: 194"; - rLen = 0; - rLoc = 6441; - rType = 0; - vrLen = 2542; - vrLoc = 4766; - }; - 5F41F82D17B00FED00C5FD90 /* PBXBookmark */ = { - isa = PBXBookmark; - fRef = 5FCC1D091595000F0003D0E9 /* conversions.hpp */; - }; - 5F41F83317B00FED00C5FD90 /* PBXBookmark */ = { - isa = PBXBookmark; - fRef = 5F9EBAAC1593F87D0092F7E2 /* chifilenames.hpp */; - }; - 5F41F85017B010A400C5FD90 /* PBXBookmark */ = { - isa = PBXBookmark; - fRef = 5F7A10471589266800748D0D /* memoryshard.hpp */; - }; - 5F41F87017B012BB00C5FD90 /* PBXBookmark */ = { - isa = PBXBookmark; - fRef = 5F7A102F1589266800748D0D /* graph_objects.hpp */; - }; 5F41F89817B0157600C5FD90 /* PBXTextBookmark */ = { isa = PBXTextBookmark; fRef = 5F41F7A917B006AE00C5FD90 /* kwaymerge.hpp */; @@ -348,122 +288,6 @@ vrLen = 2451; vrLoc = 1099; }; - 5F41F89D17B0157600C5FD90 /* orderbydegree.hpp */ = { - isa = PBXFileReference; - lastKnownFileType = sourcecode.cpp.h; - name = orderbydegree.hpp; - path = ../src/preprocessing/util/orderbydegree.hpp; - sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1069, 2041}}"; - sepNavSelRange = "{1712, 0}"; - sepNavVisRange = "{403, 1763}"; - }; - }; - 5F41F8A117B0157600C5FD90 /* PBXBookmark */ = { - isa = PBXBookmark; - fRef = 5F7A10471589266800748D0D /* memoryshard.hpp */; - }; - 5F41F8B317B0169E00C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5F41F89D17B0157600C5FD90 /* orderbydegree.hpp */; - name = "orderbydegree.hpp: 134"; - rLen = 0; - rLoc = 5452; - rType = 0; - vrLen = 3367; - vrLoc = 2369; - }; - 5F41F8B417B0169E00C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5FCC2104159DEC0E0003D0E9 /* trianglecounting.cpp */; - name = "trianglecounting.cpp: 456"; - rLen = 0; - rLoc = 15995; - rType = 0; - vrLen = 2818; - vrLoc = 14382; - }; - 5F41F8BE17B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5FCC2104159DEC0E0003D0E9 /* trianglecounting.cpp */; - name = "trianglecounting.cpp: 456"; - rLen = 0; - rLoc = 15995; - rType = 0; - vrLen = 1128; - vrLoc = 3146; - }; - 5F41F8BF17B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5F7A10471589266800748D0D /* memoryshard.hpp */; - name = "memoryshard.hpp: 1"; - rLen = 0; - rLoc = 0; - rType = 0; - vrLen = 2075; - vrLoc = 950; - }; - 5F41F8C017B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5F7A10471589266800748D0D /* memoryshard.hpp */; - name = "memoryshard.hpp: 1"; - rLen = 0; - rLoc = 0; - rType = 0; - vrLen = 3430; - vrLoc = 12496; - }; - 5F41F8C117B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5F7A102F1589266800748D0D /* graph_objects.hpp */; - name = "graph_objects.hpp: 241"; - rLen = 0; - rLoc = 7054; - rType = 0; - vrLen = 2501; - vrLoc = 6767; - }; - 5F41F8C217B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5FCC1D091595000F0003D0E9 /* conversions.hpp */; - name = "conversions.hpp: 502"; - rLen = 0; - rLoc = 18199; - rType = 0; - vrLen = 2370; - vrLoc = 17609; - }; - 5F41F8C317B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5FCC2104159DEC0E0003D0E9 /* trianglecounting.cpp */; - name = "trianglecounting.cpp: 306"; - rLen = 41; - rLoc = 9808; - rType = 0; - vrLen = 2268; - vrLoc = 8717; - }; - 5F41F8C417B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5F9EBAAC1593F87D0092F7E2 /* chifilenames.hpp */; - name = "chifilenames.hpp: 358"; - rLen = 21; - rLoc = 12998; - rType = 0; - vrLen = 2515; - vrLoc = 11046; - }; - 5F41F8C517B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5F41F79E17B0068800C5FD90 /* kwaymerge.hpp */; - name = "kwaymerge.hpp: 75"; - rLen = 0; - rLoc = 1904; - rType = 0; - vrLen = 1455; - vrLoc = 1143; - }; 5F54B84915FD4D9F00B3842C /* test_dynamicedata */ = { activeExec = 0; executables = ( @@ -570,6 +394,11 @@ name = sharder.hpp; path = ../src/preprocessing/sharder.hpp; sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1181, 15366}}"; + sepNavSelRange = "{3686, 4}"; + sepNavVisRange = "{18924, 3033}"; + }; }; 5F7A0FE11589160900748D0D /* Source Control */ = { isa = PBXSourceControlManager; @@ -927,11 +756,6 @@ name = conversions.hpp; path = ../src/preprocessing/conversions.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {803, 8762}}"; - sepNavSelRange = "{18199, 0}"; - sepNavVisRange = "{17609, 2227}"; - }; }; 5FCC1DD71599194B0003D0E9 /* graphchi_basic_includes.hpp */ = { uiCtxt = { @@ -1142,9 +966,9 @@ }; 5FCC2104159DEC0E0003D0E9 /* trianglecounting.cpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {901, 6227}}"; + sepNavIntBoundsRect = "{{0, 0}, {964, 6409}}"; sepNavSelRange = "{9808, 41}"; - sepNavVisRange = "{8728, 2235}"; + sepNavVisRange = "{8487, 2735}"; sepNavWindowFrame = "{{978, 46}, {827, 866}}"; }; }; @@ -1216,11 +1040,6 @@ name = graph_objects.hpp; path = ../src/api/graph_objects.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1265, 4862}}"; - sepNavSelRange = "{7054, 0}"; - sepNavVisRange = "{6767, 2322}"; - }; }; 5FCC229215A1EC330003D0E9 /* als_graphlab.cpp */ = { uiCtxt = { @@ -1381,10 +1200,24 @@ hitCount = 0; ignoreCount = 0; lineNumber = 267; - modificationTime = 397418581.107948; + modificationTime = 397418581.1079478; originalNumberOfMultipleMatches = 1; state = 0; }; + 5FF7741517B170E300F84939 /* PBXBookmark */ = { + isa = PBXBookmark; + fRef = 5FCC2104159DEC0E0003D0E9 /* trianglecounting.cpp */; + }; + 5FF7741617B170E300F84939 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC2104159DEC0E0003D0E9 /* trianglecounting.cpp */; + name = "trianglecounting.cpp: 306"; + rLen = 41; + rLoc = 9808; + rType = 0; + vrLen = 2735; + vrLoc = 8487; + }; 8DD76F620486A84900D96B5E /* pagerank */ = { activeExec = 0; executables = ( diff --git a/graphchi_xcode/graphchi_xcode.xcodeproj/project.pbxproj b/graphchi_xcode/graphchi_xcode.xcodeproj/project.pbxproj index c5f95fa0..df5e0602 100644 --- a/graphchi_xcode/graphchi_xcode.xcodeproj/project.pbxproj +++ b/graphchi_xcode/graphchi_xcode.xcodeproj/project.pbxproj @@ -413,6 +413,7 @@ 5FCC242B15A378DC0003D0E9 /* streaming_pagerank.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = streaming_pagerank.cpp; sourceTree = ""; }; 5FCC243415A378E70003D0E9 /* streaming_pagerank */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = streaming_pagerank; sourceTree = BUILT_PRODUCTS_DIR; }; 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = dynamicblock.hpp; sourceTree = ""; }; + 5FF7749917B3E1A200F84939 /* sharder.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = sharder.hpp; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -888,6 +889,7 @@ 5FCC1CD81594F1890003D0E9 /* sharder.hpp */, 5FCC1D041594FADE0003D0E9 /* sharder_basic.cpp */, 5F74B04F15D6CEE200ED3EA9 /* blocksplitter.cpp */, + 5FF7749817B3E1A200F84939 /* dynamicdata */, 5F41F80217B00BC600C5FD90 /* util */, ); path = preprocessing; @@ -1157,6 +1159,14 @@ path = graphlab2_1_GAS_api; sourceTree = ""; }; + 5FF7749817B3E1A200F84939 /* dynamicdata */ = { + isa = PBXGroup; + children = ( + 5FF7749917B3E1A200F84939 /* sharder.hpp */, + ); + path = dynamicdata; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ diff --git a/src/preprocessing/conversions.hpp b/src/preprocessing/conversions.hpp index 7733166e..a7f5b3f0 100644 --- a/src/preprocessing/conversions.hpp +++ b/src/preprocessing/conversions.hpp @@ -390,8 +390,8 @@ namespace graphchi { std::string line; // current line // handle header line - int n; // number of nodes - int m; // number of edges + int n=0; // number of nodes + int m=0; // number of edges int weighted; // indicates weight scheme: if (std::getline(graphFile, line)) { diff --git a/src/preprocessing/dynamicdata/sharder.hpp b/src/preprocessing/dynamicdata/sharder.hpp new file mode 100644 index 00000000..15f10019 --- /dev/null +++ b/src/preprocessing/dynamicdata/sharder.hpp @@ -0,0 +1,1145 @@ +/** + * @file + * @author Aapo Kyrola + * @version 1.0 + * + * @section LICENSE + * + * Copyright [2012] [Aapo Kyrola, Guy Blelloch, Carlos Guestrin / Carnegie Mellon University] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + + * + * @section DESCRIPTION + * + * Sharder converts a graph into shards which the GraphChi engine + * can process. + */ + +/** + * @section TODO + * Change all C-style IO to Unix-style IO. + */ + + +#ifndef GRAPHCHI_SHARDER_DEF +#define GRAPHCHI_SHARDER_DEF + + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "api/chifilenames.hpp" +#include "api/graphchi_context.hpp" +#include "graphchi_types.hpp" +#include "io/stripedio.hpp" +#include "logger/logger.hpp" +#include "engine/auxdata/degree_data.hpp" +#include "metrics/metrics.hpp" +#include "metrics/reps/basic_reporter.hpp" +#include "shards/memoryshard.hpp" +#include "shards/slidingshard.hpp" +#include "output/output.hpp" +#include "util/ioutil.hpp" +#include "util/radixSort.hpp" +#include "util/kwaymerge.hpp" +#ifdef DYNAMICEDATA +#include "util/qsort.hpp" +#endif +namespace graphchi { + template class sharded_graph_output; + + +#define SHARDER_BUFSIZE (64 * 1024 * 1024) + + enum ProcPhase { COMPUTE_INTERVALS=1, SHOVEL=2 }; + + template + class DuplicateEdgeFilter { + public: + virtual bool acceptFirst(EdgeDataType& first, EdgeDataType& second) = 0; + }; + + + template + struct edge_with_value { + vid_t src; + vid_t dst; + EdgeDataType value; + +#ifdef DYNAMICEDATA + // For dynamic edge data, we need to know if the value needs to be added + // to the vector, or are we storing an empty vector. + bool is_chivec_value; + uint16_t valindex; +#endif + edge_with_value() {} + + edge_with_value(vid_t src, vid_t dst, EdgeDataType value) : src(src), dst(dst), value(value) { +#ifdef DYNAMICEDATA + is_chivec_value = false; + valindex = 0; +#endif + } + + // Order primarily by dst, then by src + bool operator< (edge_with_value &x2) { + return (dst < x2.dst); + } + + + bool stopper() { return src == 0 && dst == 0; } + }; + + template + bool edge_t_src_less(const edge_with_value &a, const edge_with_value &b) { + if (a.src == b.src) { +#ifdef DYNAMICEDATA + if (a.dst == b.dst) { + return a.valindex < b.valindex; + } +#endif + return a.dst < b.dst; + } + return a.src < b.src; + } + + template + bool edge_t_dst_less(const edge_with_value &a, const edge_with_value &b) { + return a.dst < b.dst; + } + + template + struct dstF {inline vid_t operator() (edge_with_value a) {return a.dst;} }; + + template + struct srcF {inline vid_t operator() (edge_with_value a) {return a.src;} }; + + + + template + struct shard_flushinfo { + std::string shovelname; + size_t numedges; + edge_with_value * buffer; + vid_t max_vertex; + + shard_flushinfo(std::string shovelname, vid_t max_vertex, size_t numedges, edge_with_value * buffer) : + shovelname(shovelname), numedges(numedges), buffer(buffer), max_vertex(max_vertex) {} + + void flush() { + /* Sort */ + // TODO: remove duplicates here! + logstream(LOG_INFO) << "Sorting shovel: " << shovelname << ", max:" << max_vertex << std::endl; + iSort(buffer, (intT)numedges, (intT)max_vertex, dstF()); + logstream(LOG_INFO) << "Sort done." << shovelname << std::endl; + int f = open(shovelname.c_str(), O_WRONLY | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); + writea(f, buffer, numedges * sizeof(edge_with_value)); + close(f); + free(buffer); + } + }; + + // Run in a thread + template + static void * shard_flush_run(void * _info) { + shard_flushinfo * task = (shard_flushinfo*)_info; + task->flush(); + return NULL; + } + + + template + struct shovel_merge_source : public merge_source > { + + size_t bufsize_bytes; + size_t bufsize_edges; + std::string shovelfile; + size_t idx; + size_t bufidx; + edge_with_value * buffer; + int f; + size_t numedges; + + shovel_merge_source(size_t bufsize_bytes, std::string shovelfile) : bufsize_bytes(bufsize_bytes), + shovelfile(shovelfile), idx(0), bufidx(0) { + assert(bufsize_bytes % sizeof(edge_with_value) == 0); + f = open(shovelfile.c_str(), O_RDONLY); + + if (f < 0) { + logstream(LOG_ERROR) << "Could not open shovel file: " << shovelfile << std::endl; + printf("Error: %d, %s\n", errno, strerror(errno)); + } + + assert(f>=0); + + buffer = (edge_with_value *) malloc(bufsize_bytes); + numedges = (get_filesize(shovelfile) / sizeof(edge_with_value )); + bufsize_edges = (bufsize_bytes / sizeof(edge_with_value)); + load_next(); + } + + virtual ~shovel_merge_source() { + if (buffer != NULL) free(buffer); + buffer = NULL; + } + + void finish() { + close(f); + remove(shovelfile.c_str()); + + free(buffer); + buffer = NULL; + } + + void load_next() { + size_t len = std::min(bufsize_bytes, ((numedges - idx) * sizeof(edge_with_value))); + preada(f, buffer, len, idx * sizeof(edge_with_value)); + bufidx = 0; + } + + bool has_more() { + return idx < numedges; + } + + edge_with_value next() { + if (bufidx == bufsize_edges) { + load_next(); + } + idx++; + if (idx == numedges) { + edge_with_value x = buffer[bufidx++]; + finish(); + return x; + } + return buffer[bufidx++]; + } + }; + + template + class sharder : public merge_sink > { + + typedef edge_with_value edge_t; + + protected: + std::string basefilename; + + vid_t max_vertex_id; + + /* Sharding */ + int nshards; + std::vector< std::pair > intervals; + + int phase; + + int vertexchunk; + size_t nedges; + std::string prefix; + + int compressed_block_size; + + int * bufptrs; + size_t bufsize; + size_t edgedatasize; + size_t ebuffer_size; + size_t edges_per_block; + + vid_t filter_max_vertex; + + DuplicateEdgeFilter * duplicate_edge_filter; + + bool no_edgevalues; +#ifdef DYNAMICEDATA + edge_t last_added_edge; +#endif + + metrics m; + + + size_t curshovel_idx; + size_t shovelsize; + int numshovels; + size_t shoveled_edges; + bool shovel_sorted; + edge_with_value * curshovel_buffer; + std::vector shovelthreads; + + public: + + sharder(std::string basefilename) : basefilename(basefilename), m("sharder") { + + edgedatasize = sizeof(FinalEdgeDataType); + no_edgevalues = false; + compressed_block_size = 4096 * 1024; + filter_max_vertex = 0; + curshovel_buffer = NULL; + while (compressed_block_size % sizeof(FinalEdgeDataType) != 0) compressed_block_size++; + edges_per_block = compressed_block_size / sizeof(FinalEdgeDataType); + duplicate_edge_filter = NULL; + } + + + virtual ~sharder() { + if (curshovel_buffer == NULL) free(curshovel_buffer); + } + + void set_duplicate_filter(DuplicateEdgeFilter * filter) { + this->duplicate_edge_filter = filter; + } + + void set_max_vertex_id(vid_t maxid) { + filter_max_vertex = maxid; + } + + void set_no_edgevalues() { + no_edgevalues = true; + } + + /** + * Call to start a preprocessing session. + */ + void start_preprocessing() { + m.start_time("preprocessing"); + numshovels = 0; + shovelsize = (1024l * 1024l * size_t(get_option_int("membudget_mb", 1024)) / 4l / sizeof(edge_with_value)); + curshovel_idx = 0; + + logstream(LOG_INFO) << "Starting preprocessing, shovel size: " << shovelsize << std::endl; + + curshovel_buffer = (edge_with_value *) calloc(shovelsize, sizeof(edge_with_value)); + + assert(curshovel_buffer != NULL); + + shovelthreads.clear(); + + /* Write the maximum vertex id place holder - to be filled later */ + max_vertex_id = 0; + shoveled_edges = 0; + } + + /** + * Call to finish the preprocessing session. + */ + void end_preprocessing() { + m.stop_time("preprocessing"); + flush_shovel(false); + } + + void flush_shovel(bool async=true) { + /* Flush in separate thread unless the last one */ + shard_flushinfo * flushinfo = new shard_flushinfo(shovel_filename(numshovels), max_vertex_id, curshovel_idx, curshovel_buffer); + + if (!async) { + curshovel_buffer = NULL; + flushinfo->flush(); + + /* Wait for threads to finish */ + logstream(LOG_INFO) << "Waiting shoveling threads..." << std::endl; + for(int i=0; i < (int)shovelthreads.size(); i++) { + pthread_join(shovelthreads[i], NULL); + } + } else { + if (shovelthreads.size() > 2) { + logstream(LOG_INFO) << "Too many outstanding shoveling threads..." << std::endl; + + for(int i=0; i < (int)shovelthreads.size(); i++) { + pthread_join(shovelthreads[i], NULL); + } + shovelthreads.clear(); + } + curshovel_buffer = (edge_with_value *) calloc(shovelsize, sizeof(edge_with_value)); + pthread_t t; + int ret = pthread_create(&t, NULL, shard_flush_run, (void*)flushinfo); + shovelthreads.push_back(t); + assert(ret>=0); + } + numshovels++; + curshovel_idx=0; + } + + /** + * Add edge to be preprocessed with a value. + */ + void preprocessing_add_edge(vid_t from, vid_t to, EdgeDataType val, bool input_value=false) { + if (from == to) { + // Do not allow self-edges + return; + } + edge_with_value e(from, to, val); +#ifdef DYNAMICEDATA + e.is_chivec_value = input_value; + if (e.src == last_added_edge.src && e.dst == last_added_edge.dst) { + e.valindex = last_added_edge.valindex + 1; + } + last_added_edge = e; +#endif + curshovel_buffer[curshovel_idx++] = e; + if (curshovel_idx == shovelsize) { + flush_shovel(); + } + + max_vertex_id = std::max(std::max(from, to), max_vertex_id); + shoveled_edges++; + } + +#ifdef DYNAMICEDATA + void preprocessing_add_edge_multival(vid_t from, vid_t to, std::vector & vals) { + typename std::vector::iterator iter; + for(iter=vals.begin(); iter != vals.end(); ++iter) { + preprocessing_add_edge(from, to, *iter, true); + } + max_vertex_id = std::max(std::max(from, to), max_vertex_id); + } + +#endif + + /** + * Add edge without value to be preprocessed + */ + void preprocessing_add_edge(vid_t from, vid_t to) { + preprocessing_add_edge(from, to, EdgeDataType()); + } + + /** Buffered write function */ + template + void bwrite(int f, char * buf, char * &bufptr, T val) { + if (bufptr + sizeof(T) - buf >= SHARDER_BUFSIZE) { + writea(f, buf, bufptr - buf); + bufptr = buf; + } + *((T*)bufptr) = val; + bufptr += sizeof(T); + } + + int blockid; + + template + void edata_flush(char * buf, char * bufptr, std::string & shard_filename, size_t totbytes) { + int len = (int) (bufptr - buf); + + m.start_time("edata_flush"); + + std::string block_filename = filename_shard_edata_block(shard_filename, blockid, compressed_block_size); + int f = open(block_filename.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); + write_compressed(f, buf, len); + close(f); + + m.stop_time("edata_flush"); + + +#ifdef DYNAMICEDATA + // Write block's uncompressed size + write_block_uncompressed_size(block_filename, len); + +#endif + + blockid++; + } + + template + void bwrite_edata(char * &buf, char * &bufptr, T val, size_t & totbytes, std::string & shard_filename, + size_t & edgecounter) { + if (no_edgevalues) return; + + if (edgecounter == edges_per_block) { + edata_flush(buf, bufptr, shard_filename, totbytes); + bufptr = buf; + edgecounter = 0; + } + + // Check if buffer is big enough + if (bufptr - buf + sizeof(T) > ebuffer_size) { + ebuffer_size *= 2; + logstream(LOG_DEBUG) << "Increased buffer size to: " << ebuffer_size << std::endl; + size_t ptroff = bufptr - buf; // Remember the offset + buf = (char *) realloc(buf, ebuffer_size); + bufptr = buf + ptroff; + } + + totbytes += sizeof(T); + *((T*)bufptr) = val; + bufptr += sizeof(T); + } + + + /** + * Executes sharding. + * @param nshards_string the number of shards as a number, or "auto" for automatic determination + */ + int execute_sharding(std::string nshards_string) { + m.start_time("execute_sharding"); + + determine_number_of_shards(nshards_string); + write_shards(); + + m.stop_time("execute_sharding"); + + /* Print metrics */ + basic_reporter basicrep; + m.report(basicrep); + + return nshards; + } + + /** + * Sharding. This code might be hard to read - modify very carefully! + */ + protected: + + virtual void determine_number_of_shards(std::string nshards_string) { + if (nshards_string.find("auto") != std::string::npos || nshards_string == "0") { + logstream(LOG_INFO) << "Determining number of shards automatically." << std::endl; + + int membudget_mb = get_option_int("membudget_mb", 1024); + logstream(LOG_INFO) << "Assuming available memory is " << membudget_mb << " megabytes. " << std::endl; + logstream(LOG_INFO) << " (This can be defined with configuration parameter 'membudget_mb')" << std::endl; + + size_t numedges = shoveled_edges; + + double max_shardsize = membudget_mb * 1024. * 1024. / 8; + logstream(LOG_INFO) << "Determining maximum shard size: " << (max_shardsize / 1024. / 1024.) << " MB." << std::endl; + + nshards = (int) ( 1 + (numedges * sizeof(FinalEdgeDataType) / max_shardsize) + 0.5); + +#ifdef DYNAMICEDATA + // For dynamic edge data, more working memory is needed, thus the number of shards is larger. + nshards = (int) ( 2 + 4 * (numedges * sizeof(FinalEdgeDataType) / max_shardsize) + 0.5); +#endif + + } else { + nshards = atoi(nshards_string.c_str()); + } + assert(nshards > 0); + logstream(LOG_INFO) << "Number of shards to be created: " << nshards << std::endl; + } + + + protected: + + void one_shard_intervals() { + assert(nshards == 1); + std::string fname = filename_intervals(basefilename, nshards); + FILE * f = fopen(fname.c_str(), "w"); + intervals.push_back(std::pair(0, max_vertex_id)); + fprintf(f, "%u\n", max_vertex_id); + fclose(f); + + /* Write meta-file with the number of vertices */ + std::string numv_filename = basefilename + ".numvertices"; + f = fopen(numv_filename.c_str(), "w"); + fprintf(f, "%u\n", 1 + max_vertex_id); + fclose(f); + + assert(nshards == (int)intervals.size()); + } + + + std::string shovel_filename(int idx) { + std::stringstream ss; + ss << basefilename << sizeof(EdgeDataType) << "." << idx << ".shovel"; + return ss.str(); + } + + + int lastpart; + + + + + degree * degrees; + + virtual void finish_shard(int shard, edge_t * shovelbuf, size_t shovelsize) { + m.start_time("shard_final"); + blockid = 0; + size_t edgecounter = 0; + + logstream(LOG_INFO) << "Starting final processing for shard: " << shard << std::endl; + + std::string fname = filename_shard_adj(basefilename, shard, nshards); + std::string edfname = filename_shard_edata(basefilename, shard, nshards); + std::string edblockdirname = dirname_shard_edata_block(edfname, compressed_block_size); + + /* Make the block directory */ + if (!no_edgevalues) + mkdir(edblockdirname.c_str(), 0777); + size_t numedges = shovelsize / sizeof(edge_t); + + logstream(LOG_DEBUG) << "Shovel size:" << shovelsize << " edges: " << numedges << std::endl; + + m.start_time("finish_shard.sort"); +#ifndef DYNAMICEDATA + iSort(shovelbuf, (int)numedges, max_vertex_id, srcF()); +#else + quickSort(shovelbuf, (int)numedges, edge_t_src_less); +#endif + m.stop_time("finish_shard.sort"); + + // Remove duplicates + if (duplicate_edge_filter != NULL && numedges > 0) { + edge_t * tmpbuf = (edge_t*) calloc(numedges, sizeof(edge_t)); + size_t i = 1; + tmpbuf[0] = shovelbuf[0]; + for(size_t j=1; jacceptFirst(cur.value, prev.value)) { + // Replace the edge with the newer one + tmpbuf[i - 1] = cur; + } + } else { + tmpbuf[i++] = cur; + } + } + numedges = i; + logstream(LOG_DEBUG) << "After duplicate elimination: " << numedges << " edges" << std::endl; + free(shovelbuf); + shovelbuf = tmpbuf; tmpbuf = NULL; + } + + // Create the final file + int f = open(fname.c_str(), O_WRONLY | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); + if (f < 0) { + logstream(LOG_ERROR) << "Could not open " << fname << " error: " << strerror(errno) << std::endl; + } + assert(f >= 0); + int trerr = ftruncate(f, 0); + assert(trerr == 0); + + char * buf = (char*) malloc(SHARDER_BUFSIZE); + char * bufptr = buf; + + char * ebuf = (char*) malloc(compressed_block_size); + ebuffer_size = compressed_block_size; + char * ebufptr = ebuf; + + vid_t curvid=0; +#ifdef DYNAMICEDATA + vid_t lastdst = 0xffffffff; + int jumpover = 0; + size_t num_uniq_edges = 0; + size_t last_edge_count = 0; +#endif + size_t istart = 0; + size_t tot_edatabytes = 0; + for(size_t i=0; i <= numedges; i++) { + if (i % 10000000 == 0) logstream(LOG_DEBUG) << i << " / " << numedges << std::endl; +#ifdef DYNAMICEDATA + i += jumpover; // With dynamic values, there might be several values for one edge, and thus the edge repeated in the data. + jumpover = 0; +#endif //DYNAMICEDATA + edge_t edge = (i < numedges ? shovelbuf[i] : edge_t(0, 0, EdgeDataType())); // Last "element" is a stopper + +#ifdef DYNAMICEDATA + + if (lastdst == edge.dst && edge.src == curvid) { + // Currently not supported + logstream(LOG_ERROR) << "Duplicate edge in the stream - aborting" << std::endl; + assert(false); + } + lastdst = edge.dst; +#endif + + if (!edge.stopper()) { +#ifndef DYNAMICEDATA + bwrite_edata(ebuf, ebufptr, FinalEdgeDataType(edge.value), tot_edatabytes, edfname, edgecounter); +#else + /* If we have dynamic edge data, we need to write the header of chivector - if there are edge values */ + if (edge.is_chivec_value) { + // Need to check how many values for this edge + int count = 1; + while(shovelbuf[i + count].valindex == count) { count++; } + + assert(count < 32768); + + typename chivector::sizeword_t szw; + ((uint16_t *) &szw)[0] = (uint16_t)count; // Sizeword with length and capacity = count + ((uint16_t *) &szw)[1] = (uint16_t)count; + bwrite_edata::sizeword_t>(ebuf, ebufptr, szw, tot_edatabytes, edfname, edgecounter); + for(int j=0; j < count; j++) { + bwrite_edata(ebuf, ebufptr, EdgeDataType(shovelbuf[i + j].value), tot_edatabytes, edfname, edgecounter); + } + jumpover = count - 1; // Jump over + } else { + // Just write size word with zero + bwrite_edata(ebuf, ebufptr, 0, tot_edatabytes, edfname, edgecounter); + } + num_uniq_edges++; + +#endif + edgecounter++; // Increment edge counter here --- notice that dynamic edata case makes two or more calls to bwrite_edata before incrementing + } + if (degrees != NULL && edge.src != edge.dst) { + degrees[edge.src].outdegree++; + degrees[edge.dst].indegree++; + } + + if ((edge.src != curvid) || edge.stopper()) { + // New vertex +#ifndef DYNAMICEDATA + size_t count = i - istart; +#else + size_t count = num_uniq_edges - 1 - last_edge_count; + last_edge_count = num_uniq_edges - 1; + if (edge.stopper()) count++; +#endif + assert(count>0 || curvid==0); + if (count>0) { + if (count < 255) { + uint8_t x = (uint8_t)count; + bwrite(f, buf, bufptr, x); + } else { + bwrite(f, buf, bufptr, 0xff); + bwrite(f, buf, bufptr, (uint32_t)count); + } + } + +#ifndef DYNAMICEDATA + for(size_t j=istart; j < i; j++) { + bwrite(f, buf, bufptr, shovelbuf[j].dst); + } +#else + // Special dealing with dynamic edata because some edges can be present multiple + // times in the shovel. + for(size_t j=istart; j < i; j++) { + if (j == istart || shovelbuf[j - 1].dst != shovelbuf[j].dst) { + bwrite(f, buf, bufptr, shovelbuf[j].dst); + } + } +#endif + istart = i; +#ifdef DYNAMICEDATA + istart += jumpover; +#endif + + // Handle zeros + if (!edge.stopper()) { + if (edge.src - curvid > 1 || (i == 0 && edge.src>0)) { + int nz = edge.src - curvid - 1; + if (i == 0 && edge.src > 0) nz = edge.src; // border case with the first one + do { + bwrite(f, buf, bufptr, 0); + nz--; + int tnz = std::min(254, nz); + bwrite(f, buf, bufptr, (uint8_t) tnz); + nz -= tnz; + } while (nz>0); + } + } + curvid = edge.src; + } + } + + /* Flush buffers and free memory */ + writea(f, buf, bufptr - buf); + free(buf); + free(shovelbuf); + close(f); + + /* Write edata size file */ + if (!no_edgevalues) { + edata_flush(ebuf, ebufptr, edfname, tot_edatabytes); + + std::string sizefilename = edfname + ".size"; + std::ofstream ofs(sizefilename.c_str()); +#ifndef DYNAMICEDATA + ofs << tot_edatabytes; +#else + ofs << num_uniq_edges * sizeof(int); // For dynamic edge data, write the number of edges. +#endif + + ofs.close(); + } + free(ebuf); + + m.stop_time("shard_final"); + } + + /* Begin: Kway -merge sink interface */ + + size_t edges_per_shard; + size_t cur_shard_counter; + size_t shard_capacity; + size_t sharded_edges; + int shardnum; + edge_with_value * sinkbuffer; + vid_t prevvid; + vid_t this_interval_start; + + virtual void add(edge_with_value val) { + if (cur_shard_counter >= edges_per_shard && val.dst != prevvid) { + createnextshard(); + } + + if (cur_shard_counter == shard_capacity) { + /* Really should have a way to limit shard sizes, but probably not needed in practice */ + logstream(LOG_WARNING) << "Shard " << shardnum << " overflowing! " << cur_shard_counter << " / " << shard_capacity << std::endl; + shard_capacity = (size_t) (1.2 * shard_capacity); + sinkbuffer = (edge_with_value*) realloc(sinkbuffer, shard_capacity * sizeof(edge_with_value)); + } + + sinkbuffer[cur_shard_counter++] = val; + prevvid = val.dst; + sharded_edges++; + } + + void createnextshard() { + assert(shardnum < nshards); + intervals.push_back(std::pair(this_interval_start, (shardnum == nshards - 1 ? max_vertex_id : prevvid))); + this_interval_start = prevvid + 1; + finish_shard(shardnum++, sinkbuffer, cur_shard_counter * sizeof(edge_with_value)); + sinkbuffer = (edge_with_value *) malloc(shard_capacity * sizeof(edge_with_value)); + cur_shard_counter = 0; + + // Adjust edges per hard so that it takes into account how many edges have been spilled now + logstream(LOG_INFO) << "Remaining edges: " << (shoveled_edges - sharded_edges) << " remaining shards:" << (nshards - shardnum) + << " edges per shard=" << edges_per_shard << std::endl; + if (shardnum < nshards) edges_per_shard = (shoveled_edges - sharded_edges) / (nshards - shardnum); + logstream(LOG_INFO) << "Edges per shard: " << edges_per_shard << std::endl; + + } + + virtual void done() { + createnextshard(); + if (shoveled_edges != sharded_edges) { + logstream(LOG_INFO) << "Shoveled " << shoveled_edges << " but sharded " << sharded_edges << " edges" << std::endl; + } + assert(shoveled_edges == sharded_edges); + + logstream(LOG_INFO) << "Created " << shardnum << " shards, expected: " << nshards << std::endl; + assert(shardnum <= nshards); + free(sinkbuffer); + sinkbuffer = NULL; + + /* Write intervals */ + std::string fname = filename_intervals(basefilename, nshards); + FILE * f = fopen(fname.c_str(), "w"); + + if (f == NULL) { + logstream(LOG_ERROR) << "Could not open file: " << fname << " error: " << + strerror(errno) << std::endl; + } + assert(f != NULL); + for(int i=0; i<(int)intervals.size(); i++) { + fprintf(f, "%u\n", intervals[i].second); + } + fclose(f); + + /* Write meta-file with the number of vertices */ + std::string numv_filename = basefilename + ".numvertices"; + f = fopen(numv_filename.c_str(), "w"); + fprintf(f, "%u\n", 1 + max_vertex_id); + fclose(f); + } + + /* End: Kway -merge sink interface */ + + + + /** + * Write the shard by sorting the shovel file and compressing the + * adjacency information. + * To support different shard types, override this function! + */ + virtual void write_shards() { + + size_t membudget_mb = (size_t) get_option_int("membudget_mb", 1024); + + // Check if we have enough memory to keep track + // of the vertex degrees in-memory (heuristic) + bool count_degrees_inmem = membudget_mb * 1024 * 1024 / 3 > max_vertex_id * sizeof(degree); + degrees = NULL; +#ifdef DYNAMICEDATA + if (!count_degrees_inmem) { + /* Temporary: force in-memory count of degrees because the PSW-based computation + is not yet compatible with dynamic edge data. + */ + logstream(LOG_WARNING) << "Dynamic edge data support only sharding when the vertex degrees can be computed in-memory." << std::endl; + logstream(LOG_WARNING) << "If the program gets very slow (starts swapping), the data size is too big." << std::endl; + count_degrees_inmem = true; + } +#endif + if (count_degrees_inmem) { + degrees = (degree *) calloc(1 + max_vertex_id, sizeof(degree)); + } + + // KWAY MERGE + sharded_edges = 0; + edges_per_shard = shoveled_edges / nshards + 1; + shard_capacity = edges_per_shard / 2 * 3; // Shard can go 50% over + shardnum = 0; + this_interval_start = 0; + sinkbuffer = (edge_with_value *) calloc(shard_capacity, sizeof(edge_with_value)); + logstream(LOG_INFO) << "Edges per shard: " << edges_per_shard << " nshards=" << nshards << " total: " << shoveled_edges << std::endl; + cur_shard_counter = 0; + + /* Initialize kway merge sources */ + size_t B = membudget_mb * 1024 * 1024 / 2 / numshovels; + while (B % sizeof(edge_with_value) != 0) B++; + logstream(LOG_INFO) << "Buffer size in merge phase: " << B << std::endl; + prevvid = (-1); + std::vector< merge_source > *> sources; + for(int i=0; i < numshovels; i++) { + sources.push_back(new shovel_merge_source(B, shovel_filename(i))); + } + + kway_merge > merger(sources, this); + merger.merge(); + + // Delete sources + for(int i=0; i < (int)sources.size(); i++) { + delete (shovel_merge_source *)sources[i]; + } + + + if (!count_degrees_inmem) { +#ifndef DYNAMICEDATA + // Use memory-efficient (but slower) method to create degree-data + create_degree_file(); +#endif + + } else { + std::string degreefname = filename_degree_data(basefilename); + int degreeOutF = open(degreefname.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); + if (degreeOutF < 0) { + logstream(LOG_ERROR) << "Could not create: " << degreeOutF << std::endl; + assert(degreeOutF >= 0); + } + + writea(degreeOutF, degrees, sizeof(degree) * (1 + max_vertex_id)); + free(degrees); + close(degreeOutF); + } + + } + + + typedef char dummy_t; + + typedef sliding_shard slidingshard_t; + typedef memory_shard memshard_t; + + +#ifndef DYNAMICEDATA + void create_degree_file() { + // Initialize IO + stripedio * iomgr = new stripedio(m); + std::vector sliding_shards; + + int subwindow = 5000000; + m.set("subwindow", (size_t)subwindow); + + int loadthreads = 4; + + m.start_time("degrees.runtime"); + + /* Initialize streaming shards */ + int blocksize = compressed_block_size; + + for(int p=0; p < nshards; p++) { + logstream(LOG_INFO) << "Initialize streaming shard: " << p << std::endl; + sliding_shards.push_back( + new slidingshard_t(iomgr, filename_shard_edata(basefilename, p, nshards), + filename_shard_adj(basefilename, p, nshards), intervals[p].first, + intervals[p].second, + blocksize, m, true, true)); + } + + graphchi_context ginfo; + ginfo.nvertices = 1 + intervals[nshards - 1].second; + ginfo.scheduler = NULL; + + std::string outputfname = filename_degree_data(basefilename); + + int degreeOutF = open(outputfname.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); + if (degreeOutF < 0) { + logstream(LOG_ERROR) << "Could not create: " << degreeOutF << std::endl; + } + assert(degreeOutF >= 0); + int trerr = ftruncate(degreeOutF, ginfo.nvertices * sizeof(int) * 2); + assert(trerr == 0); + if (trerr != 0) { + logstream(LOG_FATAL) << "Could not truncate!" << std::endl; + exit(0); + } + + for(int window=0; windowflush(); + + /* Load shard[window] into memory */ + memshard_t memshard(iomgr, filename_shard_edata(basefilename, window, nshards), filename_shard_adj(basefilename, window, nshards), + interval_st, interval_en, blocksize, m); + memshard.only_adjacency = true; + logstream(LOG_INFO) << "Interval: " << interval_st << " " << interval_en << std::endl; + + for(vid_t subinterval_st=interval_st; subinterval_st <= interval_en; ) { + vid_t subinterval_en = std::min(interval_en, subinterval_st + subwindow); + logstream(LOG_INFO) << "(Degree proc.) Sub-window: [" << subinterval_st << " - " << subinterval_en << "]" << std::endl; + assert(subinterval_en >= subinterval_st && subinterval_en <= interval_en); + + /* Preallocate vertices */ + metrics_entry men = m.start_time(); + int nvertices = subinterval_en - subinterval_st + 1; + std::vector< graphchi_vertex > vertices(nvertices, graphchi_vertex()); // preallocate + + + for(int i=0; i < nvertices; i++) { + vertices[i] = graphchi_vertex(subinterval_st + i, NULL, NULL, 0, 0); + vertices[i].scheduled = true; + } + + metrics_entry me = m.start_time(); + omp_set_num_threads(loadthreads); +#pragma omp parallel for + for(int p=-1; p < nshards; p++) { + if (p == (-1)) { + // if first window, now need to load the memshard + if (memshard.loaded() == false) { + memshard.load(); + } + + /* Load vertices from memshard (only inedges for now so can be done in parallel) */ + memshard.load_vertices(subinterval_st, subinterval_en, vertices); + } else { + /* Stream forward other than the window partition */ + if (p != window) { + sliding_shards[p]->read_next_vertices(nvertices, subinterval_st, vertices, false); + } + } + } + + m.stop_time(me, "stream_ahead", window); + + + metrics_entry mev = m.start_time(); + // Read first current values + + int * vbuf = (int*) malloc(nvertices * sizeof(int) * 2); + + for(int i=0; iset_offset(memshard.offset_for_stream_cont(), memshard.offset_vid_for_stream_cont(), + memshard.edata_ptr_for_stream_cont()); + } + close(degreeOutF); + m.stop_time("degrees.runtime"); + delete iomgr; + } +#endif + + template friend class sharded_graph_output; + }; // End class sharder + + + /** + * Outputs new edges into a shard - can be used from an update function + */ + template + class sharded_graph_output : public ioutput { + + sharder * sharderobj; + mutex lock; + + public: + sharded_graph_output(std::string filename, DuplicateEdgeFilter * filter = NULL) { + sharderobj = new sharder(filename); + sharderobj->set_duplicate_filter(filter); + sharderobj->start_preprocessing(); + } + + ~sharded_graph_output() { + delete sharderobj; + sharderobj = NULL; + } + + + + public: + void output_edge(vid_t from, vid_t to) { + assert(false); // Need to use the custom method + } + + + virtual void output_edge(vid_t from, vid_t to, float value) { + assert(false); // Need to use the custom method + } + + virtual void output_edge(vid_t from, vid_t to, double value) { + assert(false); // Need to use the custom method + } + + + virtual void output_edge(vid_t from, vid_t to, int value) { + assert(false); // Need to use the custom method + } + + virtual void output_edge(vid_t from, vid_t to, size_t value) { + assert(false); // Need to use the custom method + } + + void output_edgeval(vid_t from, vid_t to, ET value) { + lock.lock(); + sharderobj->preprocessing_add_edge(from, to, value); + lock.unlock(); + } + + void output_value(vid_t vid, VT value) { + assert(false); // Not used here + } + + + void close() { + } + + size_t finish_sharding() { + sharderobj->end_preprocessing(); + + sharderobj->execute_sharding("auto"); + return sharderobj->nshards; + } + + }; + +}; // namespace + + +#endif + + + diff --git a/src/preprocessing/sharder.hpp b/src/preprocessing/sharder.hpp index 15f10019..23c41575 100644 --- a/src/preprocessing/sharder.hpp +++ b/src/preprocessing/sharder.hpp @@ -26,11 +26,10 @@ * can process. */ -/** - * @section TODO - * Change all C-style IO to Unix-style IO. - */ +#ifdef DYNAMICEDATA +#include "preprocessing/dynamicdata/sharder.hpp" // Special sharder if dynamic edata used. +#else #ifndef GRAPHCHI_SHARDER_DEF #define GRAPHCHI_SHARDER_DEF @@ -62,9 +61,8 @@ #include "util/ioutil.hpp" #include "util/radixSort.hpp" #include "util/kwaymerge.hpp" -#ifdef DYNAMICEDATA -#include "util/qsort.hpp" -#endif + + namespace graphchi { template class sharded_graph_output; @@ -86,20 +84,10 @@ namespace graphchi { vid_t dst; EdgeDataType value; -#ifdef DYNAMICEDATA - // For dynamic edge data, we need to know if the value needs to be added - // to the vector, or are we storing an empty vector. - bool is_chivec_value; - uint16_t valindex; -#endif + edge_with_value() {} - edge_with_value(vid_t src, vid_t dst, EdgeDataType value) : src(src), dst(dst), value(value) { -#ifdef DYNAMICEDATA - is_chivec_value = false; - valindex = 0; -#endif - } + edge_with_value(vid_t src, vid_t dst, EdgeDataType value) : src(src), dst(dst), value(value) {} // Order primarily by dst, then by src bool operator< (edge_with_value &x2) { @@ -113,11 +101,6 @@ namespace graphchi { template bool edge_t_src_less(const edge_with_value &a, const edge_with_value &b) { if (a.src == b.src) { -#ifdef DYNAMICEDATA - if (a.dst == b.dst) { - return a.valindex < b.valindex; - } -#endif return a.dst < b.dst; } return a.src < b.src; @@ -268,10 +251,6 @@ namespace graphchi { DuplicateEdgeFilter * duplicate_edge_filter; bool no_edgevalues; -#ifdef DYNAMICEDATA - edge_t last_added_edge; -#endif - metrics m; @@ -385,13 +364,7 @@ namespace graphchi { return; } edge_with_value e(from, to, val); -#ifdef DYNAMICEDATA - e.is_chivec_value = input_value; - if (e.src == last_added_edge.src && e.dst == last_added_edge.dst) { - e.valindex = last_added_edge.valindex + 1; - } - last_added_edge = e; -#endif + curshovel_buffer[curshovel_idx++] = e; if (curshovel_idx == shovelsize) { flush_shovel(); @@ -401,16 +374,7 @@ namespace graphchi { shoveled_edges++; } -#ifdef DYNAMICEDATA - void preprocessing_add_edge_multival(vid_t from, vid_t to, std::vector & vals) { - typename std::vector::iterator iter; - for(iter=vals.begin(); iter != vals.end(); ++iter) { - preprocessing_add_edge(from, to, *iter, true); - } - max_vertex_id = std::max(std::max(from, to), max_vertex_id); - } - -#endif + /** * Add edge without value to be preprocessed @@ -444,14 +408,6 @@ namespace graphchi { close(f); m.stop_time("edata_flush"); - - -#ifdef DYNAMICEDATA - // Write block's uncompressed size - write_block_uncompressed_size(block_filename, len); - -#endif - blockid++; } @@ -519,12 +475,7 @@ namespace graphchi { logstream(LOG_INFO) << "Determining maximum shard size: " << (max_shardsize / 1024. / 1024.) << " MB." << std::endl; nshards = (int) ( 1 + (numedges * sizeof(FinalEdgeDataType) / max_shardsize) + 0.5); - -#ifdef DYNAMICEDATA - // For dynamic edge data, more working memory is needed, thus the number of shards is larger. - nshards = (int) ( 2 + 4 * (numedges * sizeof(FinalEdgeDataType) / max_shardsize) + 0.5); -#endif - + } else { nshards = atoi(nshards_string.c_str()); } @@ -586,11 +537,8 @@ namespace graphchi { logstream(LOG_DEBUG) << "Shovel size:" << shovelsize << " edges: " << numedges << std::endl; m.start_time("finish_shard.sort"); -#ifndef DYNAMICEDATA iSort(shovelbuf, (int)numedges, max_vertex_id, srcF()); -#else - quickSort(shovelbuf, (int)numedges, edge_t_src_less); -#endif + m.stop_time("finish_shard.sort"); // Remove duplicates @@ -634,59 +582,18 @@ namespace graphchi { char * ebufptr = ebuf; vid_t curvid=0; -#ifdef DYNAMICEDATA - vid_t lastdst = 0xffffffff; - int jumpover = 0; - size_t num_uniq_edges = 0; - size_t last_edge_count = 0; -#endif + size_t istart = 0; size_t tot_edatabytes = 0; for(size_t i=0; i <= numedges; i++) { if (i % 10000000 == 0) logstream(LOG_DEBUG) << i << " / " << numedges << std::endl; -#ifdef DYNAMICEDATA - i += jumpover; // With dynamic values, there might be several values for one edge, and thus the edge repeated in the data. - jumpover = 0; -#endif //DYNAMICEDATA + edge_t edge = (i < numedges ? shovelbuf[i] : edge_t(0, 0, EdgeDataType())); // Last "element" is a stopper -#ifdef DYNAMICEDATA - - if (lastdst == edge.dst && edge.src == curvid) { - // Currently not supported - logstream(LOG_ERROR) << "Duplicate edge in the stream - aborting" << std::endl; - assert(false); - } - lastdst = edge.dst; -#endif + if (!edge.stopper()) { -#ifndef DYNAMICEDATA bwrite_edata(ebuf, ebufptr, FinalEdgeDataType(edge.value), tot_edatabytes, edfname, edgecounter); -#else - /* If we have dynamic edge data, we need to write the header of chivector - if there are edge values */ - if (edge.is_chivec_value) { - // Need to check how many values for this edge - int count = 1; - while(shovelbuf[i + count].valindex == count) { count++; } - - assert(count < 32768); - - typename chivector::sizeword_t szw; - ((uint16_t *) &szw)[0] = (uint16_t)count; // Sizeword with length and capacity = count - ((uint16_t *) &szw)[1] = (uint16_t)count; - bwrite_edata::sizeword_t>(ebuf, ebufptr, szw, tot_edatabytes, edfname, edgecounter); - for(int j=0; j < count; j++) { - bwrite_edata(ebuf, ebufptr, EdgeDataType(shovelbuf[i + j].value), tot_edatabytes, edfname, edgecounter); - } - jumpover = count - 1; // Jump over - } else { - // Just write size word with zero - bwrite_edata(ebuf, ebufptr, 0, tot_edatabytes, edfname, edgecounter); - } - num_uniq_edges++; - -#endif edgecounter++; // Increment edge counter here --- notice that dynamic edata case makes two or more calls to bwrite_edata before incrementing } if (degrees != NULL && edge.src != edge.dst) { @@ -696,13 +603,8 @@ namespace graphchi { if ((edge.src != curvid) || edge.stopper()) { // New vertex -#ifndef DYNAMICEDATA size_t count = i - istart; -#else - size_t count = num_uniq_edges - 1 - last_edge_count; - last_edge_count = num_uniq_edges - 1; - if (edge.stopper()) count++; -#endif + assert(count>0 || curvid==0); if (count>0) { if (count < 255) { @@ -714,23 +616,10 @@ namespace graphchi { } } -#ifndef DYNAMICEDATA for(size_t j=istart; j < i; j++) { bwrite(f, buf, bufptr, shovelbuf[j].dst); } -#else - // Special dealing with dynamic edata because some edges can be present multiple - // times in the shovel. - for(size_t j=istart; j < i; j++) { - if (j == istart || shovelbuf[j - 1].dst != shovelbuf[j].dst) { - bwrite(f, buf, bufptr, shovelbuf[j].dst); - } - } -#endif istart = i; -#ifdef DYNAMICEDATA - istart += jumpover; -#endif // Handle zeros if (!edge.stopper()) { @@ -762,12 +651,7 @@ namespace graphchi { std::string sizefilename = edfname + ".size"; std::ofstream ofs(sizefilename.c_str()); -#ifndef DYNAMICEDATA - ofs << tot_edatabytes; -#else - ofs << num_uniq_edges * sizeof(int); // For dynamic edge data, write the number of edges. -#endif - + ofs << tot_edatabytes; ofs.close(); } free(ebuf); @@ -869,16 +753,7 @@ namespace graphchi { // of the vertex degrees in-memory (heuristic) bool count_degrees_inmem = membudget_mb * 1024 * 1024 / 3 > max_vertex_id * sizeof(degree); degrees = NULL; -#ifdef DYNAMICEDATA - if (!count_degrees_inmem) { - /* Temporary: force in-memory count of degrees because the PSW-based computation - is not yet compatible with dynamic edge data. - */ - logstream(LOG_WARNING) << "Dynamic edge data support only sharding when the vertex degrees can be computed in-memory." << std::endl; - logstream(LOG_WARNING) << "If the program gets very slow (starts swapping), the data size is too big." << std::endl; - count_degrees_inmem = true; - } -#endif + if (count_degrees_inmem) { degrees = (degree *) calloc(1 + max_vertex_id, sizeof(degree)); } @@ -913,11 +788,8 @@ namespace graphchi { if (!count_degrees_inmem) { -#ifndef DYNAMICEDATA // Use memory-efficient (but slower) method to create degree-data - create_degree_file(); -#endif - + create_degree_file(); } else { std::string degreefname = filename_degree_data(basefilename); int degreeOutF = open(degreefname.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); @@ -940,7 +812,6 @@ namespace graphchi { typedef memory_shard memshard_t; -#ifndef DYNAMICEDATA void create_degree_file() { // Initialize IO stripedio * iomgr = new stripedio(m); @@ -1061,7 +932,6 @@ namespace graphchi { m.stop_time("degrees.runtime"); delete iomgr; } -#endif template friend class sharded_graph_output; }; // End class sharder @@ -1141,5 +1011,6 @@ namespace graphchi { #endif +#endif // ifdef DYNAMICEDATA From c00432847eac87443bfe256766ae0bb39f29fe9d Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Thu, 8 Aug 2013 10:42:15 -0400 Subject: [PATCH 02/14] cleaned up --- .../graphchi_xcode.xcodeproj/akyrola.pbxuser | 51 +++- src/preprocessing/dynamicdata/sharder.hpp | 278 ++---------------- 2 files changed, 59 insertions(+), 270 deletions(-) diff --git a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser index f9e1aa5b..effa7b73 100644 --- a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser +++ b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser @@ -2,8 +2,8 @@ { 08FB7793FE84155DC02AAC07 /* Project object */ = { activeBuildConfigurationName = Debug; - activeExecutable = 5FCC2131159DF1F20003D0E9 /* trianglecounting */; - activeTarget = 5FCC2128159DF1F20003D0E9 /* trianglecounting */; + activeExecutable = 5FCB4690174AA11C0023D9D7 /* test_dynamicedata */; + activeTarget = 5F54B84915FD4D9F00B3842C /* test_dynamicedata */; addToTargets = ( ); breakpoints = ( @@ -164,7 +164,7 @@ fRef = 5F78366C15DD718A00954509 /* sharder.hpp */; name = "sharder.hpp: 658"; rLen = 0; - rLoc = 23495; + rLoc = 20994; rType = 0; vrLen = 3755; vrLoc = 21833; @@ -202,7 +202,7 @@ fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; name = "conversions.hpp: 594"; rLen = 0; - rLoc = 22012; + rLoc = 22016; rType = 0; vrLen = 3496; vrLoc = 20430; @@ -395,9 +395,9 @@ path = ../src/preprocessing/sharder.hpp; sourceTree = ""; uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1181, 15366}}"; - sepNavSelRange = "{3686, 4}"; - sepNavVisRange = "{18924, 3033}"; + sepNavIntBoundsRect = "{{0, 0}, {1167, 12870}}"; + sepNavSelRange = "{37090, 1662}"; + sepNavVisRange = "{37024, 1926}"; }; }; 5F7A0FE11589160900748D0D /* Source Control */ = { @@ -518,9 +518,9 @@ }; 5F7A10471589266800748D0D /* memoryshard.hpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {964, 5538}}"; - sepNavSelRange = "{0, 0}"; - sepNavVisRange = "{1010, 1985}"; + sepNavIntBoundsRect = "{{0, 0}, {964, 5876}}"; + sepNavSelRange = "{877, 71}"; + sepNavVisRange = "{0, 2025}"; sepNavWindowFrame = "{{898, 63}, {1022, 1026}}"; }; }; @@ -622,6 +622,21 @@ autoAttachOnCrash = 1; breakpointsEnabled = 0; configStateDict = { + "PBXLSLaunchAction-0" = { + PBXLSLaunchAction = 0; + PBXLSLaunchStartAction = 1; + PBXLSLaunchStdioStyle = 2; + PBXLSLaunchStyle = 0; + class = PBXLSRunLaunchConfig; + commandLineArgs = ( + ); + displayName = "Executable Runner"; + environment = { + }; + identifier = com.apple.Xcode.launch.runConfig; + remoteHostInfo = ""; + startActionInfo = ""; + }; }; customDataFormattersEnabled = 1; dataTipCustomDataFormattersEnabled = 1; @@ -629,6 +644,7 @@ dataTipSortType = 0; debuggerPlugin = GDBDebugging; disassemblyDisplayState = 0; + dylibVariantSuffix = ""; enableDebugStr = 1; environmentEntries = ( ); @@ -636,9 +652,12 @@ executableUserSymbolLevel = 0; libgmallocEnabled = 0; name = test_dynamicedata; + savedGlobals = { + }; showTypeColumn = 0; sourceDirectories = ( ); + startupPath = "/Users/akyrola/Projects/GraphCHI/graphchi-cpp/"; }; 5FCB4691174AA11C0023D9D7 /* gensgd */ = { isa = PBXExecutable; @@ -756,6 +775,11 @@ name = conversions.hpp; path = ../src/preprocessing/conversions.hpp; sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {964, 10283}}"; + sepNavSelRange = "{12594, 0}"; + sepNavVisRange = "{11457, 2665}"; + }; }; 5FCC1DD71599194B0003D0E9 /* graphchi_basic_includes.hpp */ = { uiCtxt = { @@ -1218,6 +1242,13 @@ vrLen = 2735; vrLoc = 8487; }; + 5FF7749917B3E1A200F84939 /* sharder.hpp */ = { + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {964, 15288}}"; + sepNavSelRange = "{950, 0}"; + sepNavVisRange = "{750, 1071}"; + }; + }; 8DD76F620486A84900D96B5E /* pagerank */ = { activeExec = 0; executables = ( diff --git a/src/preprocessing/dynamicdata/sharder.hpp b/src/preprocessing/dynamicdata/sharder.hpp index 15f10019..d403006e 100644 --- a/src/preprocessing/dynamicdata/sharder.hpp +++ b/src/preprocessing/dynamicdata/sharder.hpp @@ -23,14 +23,13 @@ * @section DESCRIPTION * * Sharder converts a graph into shards which the GraphChi engine - * can process. - */ - -/** - * @section TODO - * Change all C-style IO to Unix-style IO. + * can process. For DYNAMICEDATA */ + +#ifndef DYNAMICEDATA + error(Can be used only with DYNAMICEDATA) +#endif #ifndef GRAPHCHI_SHARDER_DEF #define GRAPHCHI_SHARDER_DEF @@ -86,19 +85,15 @@ namespace graphchi { vid_t dst; EdgeDataType value; -#ifdef DYNAMICEDATA // For dynamic edge data, we need to know if the value needs to be added // to the vector, or are we storing an empty vector. bool is_chivec_value; uint16_t valindex; -#endif edge_with_value() {} edge_with_value(vid_t src, vid_t dst, EdgeDataType value) : src(src), dst(dst), value(value) { -#ifdef DYNAMICEDATA is_chivec_value = false; valindex = 0; -#endif } // Order primarily by dst, then by src @@ -113,11 +108,9 @@ namespace graphchi { template bool edge_t_src_less(const edge_with_value &a, const edge_with_value &b) { if (a.src == b.src) { -#ifdef DYNAMICEDATA if (a.dst == b.dst) { return a.valindex < b.valindex; } -#endif return a.dst < b.dst; } return a.src < b.src; @@ -268,9 +261,7 @@ namespace graphchi { DuplicateEdgeFilter * duplicate_edge_filter; bool no_edgevalues; -#ifdef DYNAMICEDATA edge_t last_added_edge; -#endif metrics m; @@ -385,13 +376,12 @@ namespace graphchi { return; } edge_with_value e(from, to, val); -#ifdef DYNAMICEDATA + e.is_chivec_value = input_value; if (e.src == last_added_edge.src && e.dst == last_added_edge.dst) { e.valindex = last_added_edge.valindex + 1; } last_added_edge = e; -#endif curshovel_buffer[curshovel_idx++] = e; if (curshovel_idx == shovelsize) { flush_shovel(); @@ -401,8 +391,7 @@ namespace graphchi { shoveled_edges++; } -#ifdef DYNAMICEDATA - void preprocessing_add_edge_multival(vid_t from, vid_t to, std::vector & vals) { + void preprocessing_add_edge_multival(vid_t from, vid_t to, std::vector & vals) { typename std::vector::iterator iter; for(iter=vals.begin(); iter != vals.end(); ++iter) { preprocessing_add_edge(from, to, *iter, true); @@ -410,8 +399,7 @@ namespace graphchi { max_vertex_id = std::max(std::max(from, to), max_vertex_id); } -#endif - + /** * Add edge without value to be preprocessed */ @@ -446,12 +434,8 @@ namespace graphchi { m.stop_time("edata_flush"); -#ifdef DYNAMICEDATA - // Write block's uncompressed size + // Write block's uncompressed size write_block_uncompressed_size(block_filename, len); - -#endif - blockid++; } @@ -517,13 +501,9 @@ namespace graphchi { double max_shardsize = membudget_mb * 1024. * 1024. / 8; logstream(LOG_INFO) << "Determining maximum shard size: " << (max_shardsize / 1024. / 1024.) << " MB." << std::endl; - - nshards = (int) ( 1 + (numedges * sizeof(FinalEdgeDataType) / max_shardsize) + 0.5); - -#ifdef DYNAMICEDATA + // For dynamic edge data, more working memory is needed, thus the number of shards is larger. nshards = (int) ( 2 + 4 * (numedges * sizeof(FinalEdgeDataType) / max_shardsize) + 0.5); -#endif } else { nshards = atoi(nshards_string.c_str()); @@ -586,11 +566,8 @@ namespace graphchi { logstream(LOG_DEBUG) << "Shovel size:" << shovelsize << " edges: " << numedges << std::endl; m.start_time("finish_shard.sort"); -#ifndef DYNAMICEDATA - iSort(shovelbuf, (int)numedges, max_vertex_id, srcF()); -#else + quickSort(shovelbuf, (int)numedges, edge_t_src_less); -#endif m.stop_time("finish_shard.sort"); // Remove duplicates @@ -634,23 +611,18 @@ namespace graphchi { char * ebufptr = ebuf; vid_t curvid=0; -#ifdef DYNAMICEDATA vid_t lastdst = 0xffffffff; int jumpover = 0; size_t num_uniq_edges = 0; size_t last_edge_count = 0; -#endif size_t istart = 0; size_t tot_edatabytes = 0; for(size_t i=0; i <= numedges; i++) { if (i % 10000000 == 0) logstream(LOG_DEBUG) << i << " / " << numedges << std::endl; -#ifdef DYNAMICEDATA i += jumpover; // With dynamic values, there might be several values for one edge, and thus the edge repeated in the data. jumpover = 0; -#endif //DYNAMICEDATA edge_t edge = (i < numedges ? shovelbuf[i] : edge_t(0, 0, EdgeDataType())); // Last "element" is a stopper -#ifdef DYNAMICEDATA if (lastdst == edge.dst && edge.src == curvid) { // Currently not supported @@ -658,12 +630,10 @@ namespace graphchi { assert(false); } lastdst = edge.dst; -#endif if (!edge.stopper()) { -#ifndef DYNAMICEDATA - bwrite_edata(ebuf, ebufptr, FinalEdgeDataType(edge.value), tot_edatabytes, edfname, edgecounter); -#else + + /* If we have dynamic edge data, we need to write the header of chivector - if there are edge values */ if (edge.is_chivec_value) { // Need to check how many values for this edge @@ -685,8 +655,6 @@ namespace graphchi { bwrite_edata(ebuf, ebufptr, 0, tot_edatabytes, edfname, edgecounter); } num_uniq_edges++; - -#endif edgecounter++; // Increment edge counter here --- notice that dynamic edata case makes two or more calls to bwrite_edata before incrementing } if (degrees != NULL && edge.src != edge.dst) { @@ -696,13 +664,10 @@ namespace graphchi { if ((edge.src != curvid) || edge.stopper()) { // New vertex -#ifndef DYNAMICEDATA - size_t count = i - istart; -#else + size_t count = num_uniq_edges - 1 - last_edge_count; last_edge_count = num_uniq_edges - 1; if (edge.stopper()) count++; -#endif assert(count>0 || curvid==0); if (count>0) { if (count < 255) { @@ -714,11 +679,6 @@ namespace graphchi { } } -#ifndef DYNAMICEDATA - for(size_t j=istart; j < i; j++) { - bwrite(f, buf, bufptr, shovelbuf[j].dst); - } -#else // Special dealing with dynamic edata because some edges can be present multiple // times in the shovel. for(size_t j=istart; j < i; j++) { @@ -726,11 +686,8 @@ namespace graphchi { bwrite(f, buf, bufptr, shovelbuf[j].dst); } } -#endif istart = i; -#ifdef DYNAMICEDATA istart += jumpover; -#endif // Handle zeros if (!edge.stopper()) { @@ -762,12 +719,8 @@ namespace graphchi { std::string sizefilename = edfname + ".size"; std::ofstream ofs(sizefilename.c_str()); -#ifndef DYNAMICEDATA - ofs << tot_edatabytes; -#else - ofs << num_uniq_edges * sizeof(int); // For dynamic edge data, write the number of edges. -#endif - + + ofs << num_uniq_edges * sizeof(int); // For dynamic edge data, write the number of edges. ofs.close(); } free(ebuf); @@ -869,7 +822,6 @@ namespace graphchi { // of the vertex degrees in-memory (heuristic) bool count_degrees_inmem = membudget_mb * 1024 * 1024 / 3 > max_vertex_id * sizeof(degree); degrees = NULL; -#ifdef DYNAMICEDATA if (!count_degrees_inmem) { /* Temporary: force in-memory count of degrees because the PSW-based computation is not yet compatible with dynamic edge data. @@ -878,7 +830,6 @@ namespace graphchi { logstream(LOG_WARNING) << "If the program gets very slow (starts swapping), the data size is too big." << std::endl; count_degrees_inmem = true; } -#endif if (count_degrees_inmem) { degrees = (degree *) calloc(1 + max_vertex_id, sizeof(degree)); } @@ -913,11 +864,8 @@ namespace graphchi { if (!count_degrees_inmem) { -#ifndef DYNAMICEDATA // Use memory-efficient (but slower) method to create degree-data - create_degree_file(); -#endif - + create_degree_file(); } else { std::string degreefname = filename_degree_data(basefilename); int degreeOutF = open(degreefname.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); @@ -940,202 +888,12 @@ namespace graphchi { typedef memory_shard memshard_t; -#ifndef DYNAMICEDATA - void create_degree_file() { - // Initialize IO - stripedio * iomgr = new stripedio(m); - std::vector sliding_shards; - - int subwindow = 5000000; - m.set("subwindow", (size_t)subwindow); - - int loadthreads = 4; - - m.start_time("degrees.runtime"); - - /* Initialize streaming shards */ - int blocksize = compressed_block_size; - - for(int p=0; p < nshards; p++) { - logstream(LOG_INFO) << "Initialize streaming shard: " << p << std::endl; - sliding_shards.push_back( - new slidingshard_t(iomgr, filename_shard_edata(basefilename, p, nshards), - filename_shard_adj(basefilename, p, nshards), intervals[p].first, - intervals[p].second, - blocksize, m, true, true)); - } - - graphchi_context ginfo; - ginfo.nvertices = 1 + intervals[nshards - 1].second; - ginfo.scheduler = NULL; - - std::string outputfname = filename_degree_data(basefilename); - - int degreeOutF = open(outputfname.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); - if (degreeOutF < 0) { - logstream(LOG_ERROR) << "Could not create: " << degreeOutF << std::endl; - } - assert(degreeOutF >= 0); - int trerr = ftruncate(degreeOutF, ginfo.nvertices * sizeof(int) * 2); - assert(trerr == 0); - if (trerr != 0) { - logstream(LOG_FATAL) << "Could not truncate!" << std::endl; - exit(0); - } - - for(int window=0; windowflush(); - - /* Load shard[window] into memory */ - memshard_t memshard(iomgr, filename_shard_edata(basefilename, window, nshards), filename_shard_adj(basefilename, window, nshards), - interval_st, interval_en, blocksize, m); - memshard.only_adjacency = true; - logstream(LOG_INFO) << "Interval: " << interval_st << " " << interval_en << std::endl; - - for(vid_t subinterval_st=interval_st; subinterval_st <= interval_en; ) { - vid_t subinterval_en = std::min(interval_en, subinterval_st + subwindow); - logstream(LOG_INFO) << "(Degree proc.) Sub-window: [" << subinterval_st << " - " << subinterval_en << "]" << std::endl; - assert(subinterval_en >= subinterval_st && subinterval_en <= interval_en); - - /* Preallocate vertices */ - metrics_entry men = m.start_time(); - int nvertices = subinterval_en - subinterval_st + 1; - std::vector< graphchi_vertex > vertices(nvertices, graphchi_vertex()); // preallocate - - - for(int i=0; i < nvertices; i++) { - vertices[i] = graphchi_vertex(subinterval_st + i, NULL, NULL, 0, 0); - vertices[i].scheduled = true; - } - - metrics_entry me = m.start_time(); - omp_set_num_threads(loadthreads); -#pragma omp parallel for - for(int p=-1; p < nshards; p++) { - if (p == (-1)) { - // if first window, now need to load the memshard - if (memshard.loaded() == false) { - memshard.load(); - } - - /* Load vertices from memshard (only inedges for now so can be done in parallel) */ - memshard.load_vertices(subinterval_st, subinterval_en, vertices); - } else { - /* Stream forward other than the window partition */ - if (p != window) { - sliding_shards[p]->read_next_vertices(nvertices, subinterval_st, vertices, false); - } - } - } - - m.stop_time(me, "stream_ahead", window); - - - metrics_entry mev = m.start_time(); - // Read first current values - - int * vbuf = (int*) malloc(nvertices * sizeof(int) * 2); - - for(int i=0; iset_offset(memshard.offset_for_stream_cont(), memshard.offset_vid_for_stream_cont(), - memshard.edata_ptr_for_stream_cont()); - } - close(degreeOutF); - m.stop_time("degrees.runtime"); - delete iomgr; - } -#endif template friend class sharded_graph_output; }; // End class sharder - /** - * Outputs new edges into a shard - can be used from an update function - */ - template - class sharded_graph_output : public ioutput { - - sharder * sharderobj; - mutex lock; - - public: - sharded_graph_output(std::string filename, DuplicateEdgeFilter * filter = NULL) { - sharderobj = new sharder(filename); - sharderobj->set_duplicate_filter(filter); - sharderobj->start_preprocessing(); - } - - ~sharded_graph_output() { - delete sharderobj; - sharderobj = NULL; - } - - - - public: - void output_edge(vid_t from, vid_t to) { - assert(false); // Need to use the custom method - } - - - virtual void output_edge(vid_t from, vid_t to, float value) { - assert(false); // Need to use the custom method - } - - virtual void output_edge(vid_t from, vid_t to, double value) { - assert(false); // Need to use the custom method - } - - - virtual void output_edge(vid_t from, vid_t to, int value) { - assert(false); // Need to use the custom method - } - - virtual void output_edge(vid_t from, vid_t to, size_t value) { - assert(false); // Need to use the custom method - } - - void output_edgeval(vid_t from, vid_t to, ET value) { - lock.lock(); - sharderobj->preprocessing_add_edge(from, to, value); - lock.unlock(); - } - - void output_value(vid_t vid, VT value) { - assert(false); // Not used here - } - - - void close() { - } - - size_t finish_sharding() { - sharderobj->end_preprocessing(); - - sharderobj->execute_sharding("auto"); - return sharderobj->nshards; - } - - }; - + }; // namespace From bf4e9609b75614582e043e2b3b772cb8b8886827 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Sun, 11 Aug 2013 14:20:00 -0400 Subject: [PATCH 03/14] n/a --- .../graphchi_xcode.xcodeproj/akyrola.pbxuser | 130 +----------------- 1 file changed, 2 insertions(+), 128 deletions(-) diff --git a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser index 9af724e8..1c856ab8 100644 --- a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser +++ b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser @@ -85,9 +85,8 @@ PBXFileDataSource_Warnings_ColumnID, ); }; - PBXPerProjectTemplateStateSaveDate = 397514523; - PBXWorkspaceStateSaveDate = 397514523; - + PBXPerProjectTemplateStateSaveDate = 397677050; + PBXWorkspaceStateSaveDate = 397677050; }; perUserProjectItems = { 5F41F77B17B0057900C5FD90 = 5F41F77B17B0057900C5FD90 /* PBXTextBookmark */; @@ -197,7 +196,6 @@ name = kwaymerge.hpp; path = ../src/util/kwaymerge.hpp; sourceTree = ""; - }; 5F41F7B417B0073000C5FD90 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -205,7 +203,6 @@ name = "conversions.hpp: 594"; rLen = 0; rLoc = 22016; - rType = 0; vrLen = 3496; vrLoc = 20430; @@ -291,118 +288,6 @@ vrLen = 2451; vrLoc = 1099; }; - - 5F41F89D17B0157600C5FD90 /* orderbydegree.hpp */ = { - isa = PBXFileReference; - lastKnownFileType = sourcecode.cpp.h; - name = orderbydegree.hpp; - path = ../src/preprocessing/util/orderbydegree.hpp; - sourceTree = ""; - }; - 5F41F8A117B0157600C5FD90 /* PBXBookmark */ = { - isa = PBXBookmark; - fRef = 5F7A10471589266800748D0D /* memoryshard.hpp */; - }; - 5F41F8B317B0169E00C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5F41F89D17B0157600C5FD90 /* orderbydegree.hpp */; - name = "orderbydegree.hpp: 134"; - rLen = 0; - rLoc = 5452; - rType = 0; - vrLen = 3367; - vrLoc = 2369; - }; - 5F41F8B417B0169E00C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5FCC2104159DEC0E0003D0E9 /* trianglecounting.cpp */; - name = "trianglecounting.cpp: 456"; - rLen = 0; - rLoc = 15995; - rType = 0; - vrLen = 2818; - vrLoc = 14382; - }; - 5F41F8BE17B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5FCC2104159DEC0E0003D0E9 /* trianglecounting.cpp */; - name = "trianglecounting.cpp: 456"; - rLen = 0; - rLoc = 15995; - rType = 0; - vrLen = 1128; - vrLoc = 3146; - }; - 5F41F8BF17B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5F7A10471589266800748D0D /* memoryshard.hpp */; - name = "memoryshard.hpp: 1"; - rLen = 0; - rLoc = 0; - rType = 0; - vrLen = 2075; - vrLoc = 950; - }; - 5F41F8C017B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5F7A10471589266800748D0D /* memoryshard.hpp */; - name = "memoryshard.hpp: 1"; - rLen = 0; - rLoc = 0; - rType = 0; - vrLen = 3430; - vrLoc = 12496; - }; - 5F41F8C117B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5F7A102F1589266800748D0D /* graph_objects.hpp */; - name = "graph_objects.hpp: 241"; - rLen = 0; - rLoc = 7054; - rType = 0; - vrLen = 2501; - vrLoc = 6767; - }; - 5F41F8C217B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5FCC1D091595000F0003D0E9 /* conversions.hpp */; - name = "conversions.hpp: 502"; - rLen = 0; - rLoc = 18207; - rType = 0; - vrLen = 2370; - vrLoc = 17609; - }; - 5F41F8C317B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5FCC2104159DEC0E0003D0E9 /* trianglecounting.cpp */; - name = "trianglecounting.cpp: 306"; - rLen = 41; - rLoc = 9808; - rType = 0; - vrLen = 2268; - vrLoc = 8717; - }; - 5F41F8C417B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5F9EBAAC1593F87D0092F7E2 /* chifilenames.hpp */; - name = "chifilenames.hpp: 358"; - rLen = 21; - rLoc = 12998; - rType = 0; - vrLen = 2515; - vrLoc = 11046; - }; - 5F41F8C517B0170500C5FD90 /* PBXTextBookmark */ = { - isa = PBXTextBookmark; - fRef = 5F41F79E17B0068800C5FD90 /* kwaymerge.hpp */; - name = "kwaymerge.hpp: 75"; - rLen = 0; - rLoc = 1904; - rType = 0; - vrLen = 1455; - vrLoc = 1143; - }; 5F54B84915FD4D9F00B3842C /* test_dynamicedata */ = { activeExec = 0; executables = ( @@ -523,11 +408,6 @@ name = sharder.hpp; path = ../src/preprocessing/sharder.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1167, 12870}}"; - sepNavSelRange = "{37090, 1662}"; - sepNavVisRange = "{37024, 1926}"; - }; }; 5F7A0FE11589160900748D0D /* Source Control */ = { isa = PBXSourceControlManager; @@ -904,11 +784,6 @@ name = conversions.hpp; path = ../src/preprocessing/conversions.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {964, 10296}}"; - sepNavSelRange = "{12598, 0}"; - sepNavVisRange = "{11061, 2493}"; - }; }; 5FCC1DD71599194B0003D0E9 /* graphchi_basic_includes.hpp */ = { uiCtxt = { @@ -1354,7 +1229,6 @@ ignoreCount = 0; lineNumber = 267; modificationTime = 397418581.1079478; - originalNumberOfMultipleMatches = 1; state = 0; }; From 0b31527edbfbfd5e27f99b09e70b02b63fcacf64 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Tue, 13 Aug 2013 22:37:16 -0400 Subject: [PATCH 04/14] compiling state, but very likely not working --- src/api/dynamicdata/chivector.hpp | 19 +- src/graphchi_types.hpp | 5 + src/preprocessing/conversions.hpp | 8 +- src/preprocessing/dynamicdata/sharder.hpp | 302 ++++++++++------------ src/preprocessing/sharder.hpp | 5 +- src/shards/dynamicdata/dynamicblock.hpp | 4 +- src/tests/test_dynamicedata_loader.cpp | 58 +++-- src/util/kwaymerge.hpp | 2 +- 8 files changed, 203 insertions(+), 200 deletions(-) diff --git a/src/api/dynamicdata/chivector.hpp b/src/api/dynamicdata/chivector.hpp index 517d6dbc..461fea39 100644 --- a/src/api/dynamicdata/chivector.hpp +++ b/src/api/dynamicdata/chivector.hpp @@ -37,33 +37,26 @@ namespace graphchi { #define MINCAPACITY 2 - -/** - * Pool the extension parts of chi-vectors - */ -template -class extension_pool { - -}; - -template +template class chivector { uint16_t nsize; uint16_t ncapacity; + HeaderType hdr; // statically sized header T * data; std::vector * extensions; // TODO: use a more memory efficient system? public: typedef T element_type_t; + typedef HeaderType header_t; typedef uint32_t sizeword_t; chivector() { extensions = NULL; } - chivector(uint16_t sz, uint16_t cap, T * dataptr) : data(dataptr) { + chivector(uint16_t sz, uint16_t cap, HeaderType hdr, T * dataptr) : hdr(hdr), data(dataptr) { nsize = sz; ncapacity = cap; assert(cap >= nsize); @@ -77,6 +70,10 @@ class chivector { } } + HeaderType & header() { + return hdr; + } + void write(T * dest) { int sz = (int) this->size(); for(int i=0; i < sz; i++) { diff --git a/src/graphchi_types.hpp b/src/graphchi_types.hpp index e32a447f..f5587c62 100644 --- a/src/graphchi_types.hpp +++ b/src/graphchi_types.hpp @@ -52,6 +52,11 @@ namespace graphchi { } } }; + + // Empty data type + struct dummy { + }; + } diff --git a/src/preprocessing/conversions.hpp b/src/preprocessing/conversions.hpp index 0d05583b..1eab3bd6 100644 --- a/src/preprocessing/conversions.hpp +++ b/src/preprocessing/conversions.hpp @@ -54,10 +54,7 @@ #endif namespace graphchi { - - struct dummy { - }; - + template struct dummyC { operator T() { return T(); } @@ -276,7 +273,7 @@ namespace graphchi { logstream(LOG_FATAL) << "Each edge needs at least one value." << std::endl; assert(vals.size() > 0); } - sharderobj.preprocessing_add_edge_multival(from, to, vals); + sharderobj.preprocessing_add_edge_multival(from, to, dummy(), vals); } #else @@ -658,6 +655,7 @@ namespace graphchi { */ int convert_none(std::string basefilename, std::string nshards_string); int convert_none(std::string basefilename, std::string nshards_string) { + sharder sharderobj(basefilename); sharderobj.set_no_edgevalues(); diff --git a/src/preprocessing/dynamicdata/sharder.hpp b/src/preprocessing/dynamicdata/sharder.hpp index d403006e..0bef80d3 100644 --- a/src/preprocessing/dynamicdata/sharder.hpp +++ b/src/preprocessing/dynamicdata/sharder.hpp @@ -46,6 +46,7 @@ #include #include #include +#include #include "api/chifilenames.hpp" #include "api/graphchi_context.hpp" @@ -65,117 +66,136 @@ #include "util/qsort.hpp" #endif namespace graphchi { - template class sharded_graph_output; - + #define SHARDER_BUFSIZE (64 * 1024 * 1024) enum ProcPhase { COMPUTE_INTERVALS=1, SHOVEL=2 }; + - template - class DuplicateEdgeFilter { - public: - virtual bool acceptFirst(EdgeDataType& first, EdgeDataType& second) = 0; - }; - - - template + template struct edge_with_value { vid_t src; vid_t dst; - EdgeDataType value; + HeaderDataType hdr; + std::vector value; // For dynamic edge data, we need to know if the value needs to be added // to the vector, or are we storing an empty vector. bool is_chivec_value; - uint16_t valindex; edge_with_value() {} - edge_with_value(vid_t src, vid_t dst, EdgeDataType value) : src(src), dst(dst), value(value) { - is_chivec_value = false; - valindex = 0; + edge_with_value(vid_t src, vid_t dst, std::vector value) : src(src), dst(dst), value(value) { + } + + edge_with_value(vid_t src, vid_t dst, std::vector value, HeaderDataType hdr) : src(src), dst(dst), value(value), hdr(hdr) { } // Order primarily by dst, then by src - bool operator< (edge_with_value &x2) { + bool operator< (edge_with_value &x2) { return (dst < x2.dst); } + void reade(int f) { + read(f, &src, sizeof(vid_t)); + read(f, &dst, sizeof(vid_t)); + uint16_t nvalues; + read(f, &nvalues, sizeof(uint16_t)); + value.resize(nvalues); + read(f, &value[0], sizeof(VectorElementType) * nvalues); + read(f, &hdr, sizeof(HeaderDataType)); + } + + void writee(int f) { + writea(f, &src, sizeof(vid_t)); + writea(f, &dst, sizeof(vid_t)); + uint16_t nvalues = value.size(); + assert(value.size() < 1<<16); + writea(f, &nvalues, sizeof(uint16_t)); + writea(f, &value[0], sizeof(VectorElementType) * nvalues); + writea(f, &hdr, sizeof(HeaderDataType)); + } + bool stopper() { return src == 0 && dst == 0; } }; - template - bool edge_t_src_less(const edge_with_value &a, const edge_with_value &b) { + template + bool edge_t_src_less(const edge_with_value &a, const edge_with_value &b) { if (a.src == b.src) { - if (a.dst == b.dst) { - return a.valindex < b.valindex; - } return a.dst < b.dst; } return a.src < b.src; } - template - bool edge_t_dst_less(const edge_with_value &a, const edge_with_value &b) { + template + bool edge_t_dst_less(const edge_with_value &a, const edge_with_value &b) { return a.dst < b.dst; } - template - struct dstF {inline vid_t operator() (edge_with_value a) {return a.dst;} }; + template + struct dstF {inline vid_t operator() (edge_with_value a) {return a.dst;} }; - template - struct srcF {inline vid_t operator() (edge_with_value a) {return a.src;} }; + template + struct srcF {inline vid_t operator() (edge_with_value a) {return a.src;} }; - template + template struct shard_flushinfo { std::string shovelname; size_t numedges; - edge_with_value * buffer; + edge_with_value * buffer; vid_t max_vertex; - shard_flushinfo(std::string shovelname, vid_t max_vertex, size_t numedges, edge_with_value * buffer) : + shard_flushinfo(std::string shovelname, vid_t max_vertex, size_t numedges, edge_with_value * buffer) : shovelname(shovelname), numedges(numedges), buffer(buffer), max_vertex(max_vertex) {} void flush() { /* Sort */ // TODO: remove duplicates here! logstream(LOG_INFO) << "Sorting shovel: " << shovelname << ", max:" << max_vertex << std::endl; - iSort(buffer, (intT)numedges, (intT)max_vertex, dstF()); + iSort(buffer, (intT)numedges, (intT)max_vertex, dstF()); logstream(LOG_INFO) << "Sort done." << shovelname << std::endl; int f = open(shovelname.c_str(), O_WRONLY | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); - writea(f, buffer, numedges * sizeof(edge_with_value)); + //writea(f, buffer, numedges * sizeof(edge_with_value)); + + writea(f, &numedges, sizeof(size_t)); + + /* Write the edges into file. This is quite a bit more complicated than with non-dynamic data... */ + for(size_t i=0; i &edge = buffer[i]; + edge.writee(f); + } + close(f); free(buffer); } }; // Run in a thread - template + template static void * shard_flush_run(void * _info) { - shard_flushinfo * task = (shard_flushinfo*)_info; + shard_flushinfo * task = (shard_flushinfo*)_info; task->flush(); return NULL; } - template - struct shovel_merge_source : public merge_source > { + template + struct shovel_merge_source : public merge_source > { size_t bufsize_bytes; - size_t bufsize_edges; std::string shovelfile; size_t idx; size_t bufidx; - edge_with_value * buffer; + std::vector > buffer; int f; size_t numedges; + size_t bufsize_edges; shovel_merge_source(size_t bufsize_bytes, std::string shovelfile) : bufsize_bytes(bufsize_bytes), shovelfile(shovelfile), idx(0), bufidx(0) { - assert(bufsize_bytes % sizeof(edge_with_value) == 0); f = open(shovelfile.c_str(), O_RDONLY); if (f < 0) { @@ -185,28 +205,28 @@ namespace graphchi { assert(f>=0); - buffer = (edge_with_value *) malloc(bufsize_bytes); - numedges = (get_filesize(shovelfile) / sizeof(edge_with_value )); - bufsize_edges = (bufsize_bytes / sizeof(edge_with_value)); + bufsize_edges = bufsize_bytes / sizeof(edge_with_value); + read(f, &numedges, sizeof(size_t)); load_next(); } - virtual ~shovel_merge_source() { - if (buffer != NULL) free(buffer); - buffer = NULL; - } + virtual ~shovel_merge_source() {} void finish() { close(f); remove(shovelfile.c_str()); - - free(buffer); - buffer = NULL; + buffer.clear(); } void load_next() { - size_t len = std::min(bufsize_bytes, ((numedges - idx) * sizeof(edge_with_value))); - preada(f, buffer, len, idx * sizeof(edge_with_value)); + size_t nread = 0; + buffer.clear(); + while(nread < bufsize_bytes) { + edge_with_value edge; + edge.reade(f); + buffer.push_back(edge); + nread += sizeof(vid_t) * 2 + sizeof(uint16_t) + sizeof(HeaderDataType) + edge.value.size() * sizeof(VectorElementType); + } bufidx = 0; } @@ -214,13 +234,13 @@ namespace graphchi { return idx < numedges; } - edge_with_value next() { + edge_with_value & next() { if (bufidx == bufsize_edges) { load_next(); } idx++; if (idx == numedges) { - edge_with_value x = buffer[bufidx++]; + edge_with_value &x = buffer[bufidx++]; finish(); return x; } @@ -228,10 +248,10 @@ namespace graphchi { } }; - template - class sharder : public merge_sink > { + template + class sharder : public merge_sink > { - typedef edge_with_value edge_t; + typedef edge_with_value edge_t; protected: std::string basefilename; @@ -258,7 +278,6 @@ namespace graphchi { vid_t filter_max_vertex; - DuplicateEdgeFilter * duplicate_edge_filter; bool no_edgevalues; edge_t last_added_edge; @@ -271,31 +290,27 @@ namespace graphchi { int numshovels; size_t shoveled_edges; bool shovel_sorted; - edge_with_value * curshovel_buffer; + edge_with_value * curshovel_buffer; std::vector shovelthreads; public: sharder(std::string basefilename) : basefilename(basefilename), m("sharder") { - edgedatasize = sizeof(FinalEdgeDataType); + edgedatasize = sizeof(VectorElementType); no_edgevalues = false; compressed_block_size = 4096 * 1024; filter_max_vertex = 0; curshovel_buffer = NULL; - while (compressed_block_size % sizeof(FinalEdgeDataType) != 0) compressed_block_size++; - edges_per_block = compressed_block_size / sizeof(FinalEdgeDataType); - duplicate_edge_filter = NULL; + while (compressed_block_size % sizeof(VectorElementType) != 0) compressed_block_size++; + edges_per_block = compressed_block_size / sizeof(VectorElementType); } virtual ~sharder() { if (curshovel_buffer == NULL) free(curshovel_buffer); } - - void set_duplicate_filter(DuplicateEdgeFilter * filter) { - this->duplicate_edge_filter = filter; - } + void set_max_vertex_id(vid_t maxid) { filter_max_vertex = maxid; @@ -311,12 +326,12 @@ namespace graphchi { void start_preprocessing() { m.start_time("preprocessing"); numshovels = 0; - shovelsize = (1024l * 1024l * size_t(get_option_int("membudget_mb", 1024)) / 4l / sizeof(edge_with_value)); + shovelsize = (1024l * 1024l * size_t(get_option_int("membudget_mb", 1024)) / 4l / sizeof(edge_with_value)); curshovel_idx = 0; logstream(LOG_INFO) << "Starting preprocessing, shovel size: " << shovelsize << std::endl; - curshovel_buffer = (edge_with_value *) calloc(shovelsize, sizeof(edge_with_value)); + curshovel_buffer = (edge_with_value *) calloc(shovelsize, sizeof(edge_with_value)); assert(curshovel_buffer != NULL); @@ -337,7 +352,7 @@ namespace graphchi { void flush_shovel(bool async=true) { /* Flush in separate thread unless the last one */ - shard_flushinfo * flushinfo = new shard_flushinfo(shovel_filename(numshovels), max_vertex_id, curshovel_idx, curshovel_buffer); + shard_flushinfo * flushinfo = new shard_flushinfo(shovel_filename(numshovels), max_vertex_id, curshovel_idx, curshovel_buffer); if (!async) { curshovel_buffer = NULL; @@ -357,9 +372,9 @@ namespace graphchi { } shovelthreads.clear(); } - curshovel_buffer = (edge_with_value *) calloc(shovelsize, sizeof(edge_with_value)); + curshovel_buffer = (edge_with_value *) calloc(shovelsize, sizeof(edge_with_value)); pthread_t t; - int ret = pthread_create(&t, NULL, shard_flush_run, (void*)flushinfo); + int ret = pthread_create(&t, NULL, shard_flush_run, (void*)flushinfo); shovelthreads.push_back(t); assert(ret>=0); } @@ -370,17 +385,15 @@ namespace graphchi { /** * Add edge to be preprocessed with a value. */ - void preprocessing_add_edge(vid_t from, vid_t to, EdgeDataType val, bool input_value=false) { + void preprocessing_add_edge(vid_t from, vid_t to, VectorElementType val, bool input_value=false) { if (from == to) { // Do not allow self-edges return; - } - edge_with_value e(from, to, val); + } + assert(!input_value); + edge_with_value e(from, to, std::vector(0)); e.is_chivec_value = input_value; - if (e.src == last_added_edge.src && e.dst == last_added_edge.dst) { - e.valindex = last_added_edge.valindex + 1; - } last_added_edge = e; curshovel_buffer[curshovel_idx++] = e; if (curshovel_idx == shovelsize) { @@ -391,12 +404,21 @@ namespace graphchi { shoveled_edges++; } - void preprocessing_add_edge_multival(vid_t from, vid_t to, std::vector & vals) { - typename std::vector::iterator iter; - for(iter=vals.begin(); iter != vals.end(); ++iter) { - preprocessing_add_edge(from, to, *iter, true); - } - max_vertex_id = std::max(std::max(from, to), max_vertex_id); + void preprocessing_add_edge_multival(vid_t from, vid_t to, HeaderDataType hdr, std::vector & vals) { + if (from == to) { + // Do not allow self-edges + return; + } + edge_with_value e(from, to, vals, hdr); + + e.is_chivec_value = true; + curshovel_buffer[curshovel_idx++] = e; + if (curshovel_idx == shovelsize) { + flush_shovel(); + } + + max_vertex_id = std::max(std::max(from, to), max_vertex_id); + shoveled_edges++; } @@ -404,7 +426,7 @@ namespace graphchi { * Add edge without value to be preprocessed */ void preprocessing_add_edge(vid_t from, vid_t to) { - preprocessing_add_edge(from, to, EdgeDataType()); + preprocessing_add_edge(from, to, VectorElementType()); } /** Buffered write function */ @@ -503,7 +525,7 @@ namespace graphchi { logstream(LOG_INFO) << "Determining maximum shard size: " << (max_shardsize / 1024. / 1024.) << " MB." << std::endl; // For dynamic edge data, more working memory is needed, thus the number of shards is larger. - nshards = (int) ( 2 + 4 * (numedges * sizeof(FinalEdgeDataType) / max_shardsize) + 0.5); + nshards = (int) ( 2 + 4 * (numedges * sizeof(VectorElementType) / max_shardsize) + 0.5); } else { nshards = atoi(nshards_string.c_str()); @@ -535,7 +557,7 @@ namespace graphchi { std::string shovel_filename(int idx) { std::stringstream ss; - ss << basefilename << sizeof(EdgeDataType) << "." << idx << ".shovel"; + ss << basefilename << sizeof(VectorElementType) << "." << idx << ".shovel"; return ss.str(); } @@ -555,7 +577,7 @@ namespace graphchi { logstream(LOG_INFO) << "Starting final processing for shard: " << shard << std::endl; std::string fname = filename_shard_adj(basefilename, shard, nshards); - std::string edfname = filename_shard_edata(basefilename, shard, nshards); + std::string edfname = filename_shard_edata(basefilename, shard, nshards); std::string edblockdirname = dirname_shard_edata_block(edfname, compressed_block_size); /* Make the block directory */ @@ -567,33 +589,9 @@ namespace graphchi { m.start_time("finish_shard.sort"); - quickSort(shovelbuf, (int)numedges, edge_t_src_less); + quickSort(shovelbuf, (int)numedges, edge_t_src_less); m.stop_time("finish_shard.sort"); - // Remove duplicates - if (duplicate_edge_filter != NULL && numedges > 0) { - edge_t * tmpbuf = (edge_t*) calloc(numedges, sizeof(edge_t)); - size_t i = 1; - tmpbuf[0] = shovelbuf[0]; - for(size_t j=1; jacceptFirst(cur.value, prev.value)) { - // Replace the edge with the newer one - tmpbuf[i - 1] = cur; - } - } else { - tmpbuf[i++] = cur; - } - } - numedges = i; - logstream(LOG_DEBUG) << "After duplicate elimination: " << numedges << " edges" << std::endl; - free(shovelbuf); - shovelbuf = tmpbuf; tmpbuf = NULL; - } - // Create the final file int f = open(fname.c_str(), O_WRONLY | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); if (f < 0) { @@ -621,8 +619,7 @@ namespace graphchi { if (i % 10000000 == 0) logstream(LOG_DEBUG) << i << " / " << numedges << std::endl; i += jumpover; // With dynamic values, there might be several values for one edge, and thus the edge repeated in the data. jumpover = 0; - edge_t edge = (i < numedges ? shovelbuf[i] : edge_t(0, 0, EdgeDataType())); // Last "element" is a stopper - + edge_t edge = (i < numedges ? shovelbuf[i] : edge_t(0, 0, std::vector())); // Last "element" is a stopper if (lastdst == edge.dst && edge.src == curvid) { // Currently not supported @@ -635,20 +632,22 @@ namespace graphchi { /* If we have dynamic edge data, we need to write the header of chivector - if there are edge values */ + bwrite_edata(ebuf, ebufptr, shovelbuf[i].hdr, tot_edatabytes, edfname, edgecounter); + if (edge.is_chivec_value) { // Need to check how many values for this edge - int count = 1; - while(shovelbuf[i + count].valindex == count) { count++; } + int count = shovelbuf[i].value.size(); assert(count < 32768); - - typename chivector::sizeword_t szw; + typename chivector::sizeword_t szw; ((uint16_t *) &szw)[0] = (uint16_t)count; // Sizeword with length and capacity = count ((uint16_t *) &szw)[1] = (uint16_t)count; - bwrite_edata::sizeword_t>(ebuf, ebufptr, szw, tot_edatabytes, edfname, edgecounter); + bwrite_edata::sizeword_t>(ebuf, ebufptr, szw, tot_edatabytes, edfname, edgecounter); for(int j=0; j < count; j++) { - bwrite_edata(ebuf, ebufptr, EdgeDataType(shovelbuf[i + j].value), tot_edatabytes, edfname, edgecounter); + bwrite_edata(ebuf, ebufptr, shovelbuf[i].value[j], tot_edatabytes, edfname, edgecounter); } + + jumpover = count - 1; // Jump over } else { // Just write size word with zero @@ -715,7 +714,7 @@ namespace graphchi { /* Write edata size file */ if (!no_edgevalues) { - edata_flush(ebuf, ebufptr, edfname, tot_edatabytes); + edata_flush(ebuf, ebufptr, edfname, tot_edatabytes); std::string sizefilename = edfname + ".size"; std::ofstream ofs(sizefilename.c_str()); @@ -735,11 +734,11 @@ namespace graphchi { size_t shard_capacity; size_t sharded_edges; int shardnum; - edge_with_value * sinkbuffer; + edge_with_value * sinkbuffer; vid_t prevvid; vid_t this_interval_start; - virtual void add(edge_with_value val) { + virtual void add(edge_with_value val) { if (cur_shard_counter >= edges_per_shard && val.dst != prevvid) { createnextshard(); } @@ -748,7 +747,7 @@ namespace graphchi { /* Really should have a way to limit shard sizes, but probably not needed in practice */ logstream(LOG_WARNING) << "Shard " << shardnum << " overflowing! " << cur_shard_counter << " / " << shard_capacity << std::endl; shard_capacity = (size_t) (1.2 * shard_capacity); - sinkbuffer = (edge_with_value*) realloc(sinkbuffer, shard_capacity * sizeof(edge_with_value)); + sinkbuffer = (edge_with_value*) realloc(sinkbuffer, shard_capacity * sizeof(edge_with_value)); } sinkbuffer[cur_shard_counter++] = val; @@ -760,8 +759,8 @@ namespace graphchi { assert(shardnum < nshards); intervals.push_back(std::pair(this_interval_start, (shardnum == nshards - 1 ? max_vertex_id : prevvid))); this_interval_start = prevvid + 1; - finish_shard(shardnum++, sinkbuffer, cur_shard_counter * sizeof(edge_with_value)); - sinkbuffer = (edge_with_value *) malloc(shard_capacity * sizeof(edge_with_value)); + finish_shard(shardnum++, sinkbuffer, cur_shard_counter * sizeof(edge_with_value)); + sinkbuffer = (edge_with_value *) malloc(shard_capacity * sizeof(edge_with_value)); cur_shard_counter = 0; // Adjust edges per hard so that it takes into account how many edges have been spilled now @@ -820,19 +819,7 @@ namespace graphchi { // Check if we have enough memory to keep track // of the vertex degrees in-memory (heuristic) - bool count_degrees_inmem = membudget_mb * 1024 * 1024 / 3 > max_vertex_id * sizeof(degree); - degrees = NULL; - if (!count_degrees_inmem) { - /* Temporary: force in-memory count of degrees because the PSW-based computation - is not yet compatible with dynamic edge data. - */ - logstream(LOG_WARNING) << "Dynamic edge data support only sharding when the vertex degrees can be computed in-memory." << std::endl; - logstream(LOG_WARNING) << "If the program gets very slow (starts swapping), the data size is too big." << std::endl; - count_degrees_inmem = true; - } - if (count_degrees_inmem) { - degrees = (degree *) calloc(1 + max_vertex_id, sizeof(degree)); - } + degrees = (degree *) calloc(1 + max_vertex_id, sizeof(degree)); // KWAY MERGE sharded_edges = 0; @@ -840,45 +827,40 @@ namespace graphchi { shard_capacity = edges_per_shard / 2 * 3; // Shard can go 50% over shardnum = 0; this_interval_start = 0; - sinkbuffer = (edge_with_value *) calloc(shard_capacity, sizeof(edge_with_value)); + sinkbuffer = (edge_with_value *) calloc(shard_capacity, sizeof(edge_with_value)); logstream(LOG_INFO) << "Edges per shard: " << edges_per_shard << " nshards=" << nshards << " total: " << shoveled_edges << std::endl; cur_shard_counter = 0; /* Initialize kway merge sources */ size_t B = membudget_mb * 1024 * 1024 / 2 / numshovels; - while (B % sizeof(edge_with_value) != 0) B++; + while (B % sizeof(edge_with_value) != 0) B++; logstream(LOG_INFO) << "Buffer size in merge phase: " << B << std::endl; prevvid = (-1); - std::vector< merge_source > *> sources; + std::vector< merge_source > *> sources; for(int i=0; i < numshovels; i++) { - sources.push_back(new shovel_merge_source(B, shovel_filename(i))); + sources.push_back(new shovel_merge_source(B, shovel_filename(i))); } - kway_merge > merger(sources, this); + kway_merge > merger(sources, this); merger.merge(); // Delete sources for(int i=0; i < (int)sources.size(); i++) { - delete (shovel_merge_source *)sources[i]; + delete (shovel_merge_source *)sources[i]; } - if (!count_degrees_inmem) { - // Use memory-efficient (but slower) method to create degree-data - create_degree_file(); - } else { - std::string degreefname = filename_degree_data(basefilename); - int degreeOutF = open(degreefname.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); - if (degreeOutF < 0) { - logstream(LOG_ERROR) << "Could not create: " << degreeOutF << std::endl; - assert(degreeOutF >= 0); - } - - writea(degreeOutF, degrees, sizeof(degree) * (1 + max_vertex_id)); - free(degrees); - close(degreeOutF); + + std::string degreefname = filename_degree_data(basefilename); + int degreeOutF = open(degreefname.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); + if (degreeOutF < 0) { + logstream(LOG_ERROR) << "Could not create: " << degreeOutF << std::endl; + assert(degreeOutF >= 0); } + writea(degreeOutF, degrees, sizeof(degree) * (1 + max_vertex_id)); + free(degrees); + close(degreeOutF); } @@ -888,8 +870,6 @@ namespace graphchi { typedef memory_shard memshard_t; - - template friend class sharded_graph_output; }; // End class sharder diff --git a/src/preprocessing/sharder.hpp b/src/preprocessing/sharder.hpp index 23c41575..da884d5a 100644 --- a/src/preprocessing/sharder.hpp +++ b/src/preprocessing/sharder.hpp @@ -204,7 +204,7 @@ namespace graphchi { return idx < numedges; } - edge_with_value next() { + edge_with_value & next() { if (bufidx == bufsize_edges) { load_next(); } @@ -374,6 +374,9 @@ namespace graphchi { shoveled_edges++; } + + + /** diff --git a/src/shards/dynamicdata/dynamicblock.hpp b/src/shards/dynamicdata/dynamicblock.hpp index 503d1eb0..cb386ce3 100644 --- a/src/shards/dynamicdata/dynamicblock.hpp +++ b/src/shards/dynamicdata/dynamicblock.hpp @@ -83,9 +83,11 @@ namespace graphchi { uint8_t * ptr = data; for(int i=0; i < nitems; i++) { assert(ptr - data <= datasize); + typename ET::header_t hdr = *((typename ET::header_t *) ptr); + ptr += sizeof(typename ET::header_t); typename ET::sizeword_t * sz = ((typename ET::sizeword_t *) ptr); ptr += sizeof(typename ET::sizeword_t); - chivecs[i] = ET(((uint16_t *)sz)[0], ((uint16_t *)sz)[1], (typename ET::element_type_t *) ptr); + chivecs[i] = ET(((uint16_t *)sz)[0], ((uint16_t *)sz)[1], hdr, (typename ET::element_type_t *) ptr); ptr += (int) ((uint16_t *)sz)[1] * sizeof(typename ET::element_type_t); } } diff --git a/src/tests/test_dynamicedata_loader.cpp b/src/tests/test_dynamicedata_loader.cpp index f9bd8450..c88edcaf 100644 --- a/src/tests/test_dynamicedata_loader.cpp +++ b/src/tests/test_dynamicedata_loader.cpp @@ -42,8 +42,15 @@ using namespace graphchi; * Type definitions. Remember to create suitable graph shards using the * Sharder-program. */ -typedef chivector VertexDataType; -typedef chivector EdgeDataType; +struct header_t { + vid_t a; + bool b; + header_t() {} + header_t(vid_t a, bool b): a(a), b(b) {} +}; + +typedef chivector VertexDataType; +typedef chivector EdgeDataType; size_t checksum = 0; size_t shouldbe = 0; @@ -60,7 +67,7 @@ struct DynamicDataLoaderTestProgram : public GraphChiProgram &vertex, graphchi_context &gcontext) { for(int i=0; i < vertex.num_edges(); i++) { - chivector * evector = vertex.edge(i)->get_vector(); + chivector * evector = vertex.edge(i)->get_vector(); assert(evector != NULL); @@ -77,13 +84,17 @@ struct DynamicDataLoaderTestProgram : public GraphChiProgramget(k) == expected); } + assert(evector->header().a == vertex.id() + vertex.edge(i)->vertex_id()); + assert(evector->header().b == gcontext.iteration %2); + evector->header().b = !evector->header().b; + lock.lock(); checksum += evector->get(0); lock.unlock(); } // Modify vertex data by adding values there */ - chivector * vvector = vertex.get_vector(); + chivector * vvector = vertex.get_vector(); int numitems = vertex.id() % 10; for(int i=0; iadd(vertex.id() * 982192l + i); // Arbitrary @@ -124,12 +135,13 @@ struct DynamicDataLoaderTestProgram : public GraphChiProgram sharderobj(filename); + sharderobj.start_preprocessing(); + std::cout << "Generating data..." << std::endl; - const char * fname = filename.c_str(); - FILE * f = fopen(fname, "w"); - set_conf("filetype", "edgelist"); + shouldbe = 0; int totalVertices = 200000; // 2 million for(int i=0; i < totalVertices; i++) { @@ -137,22 +149,29 @@ void generatedata(std::string filename) { for(int j=0; j < nedges; j++) { int dst = (totalVertices / nedges) * j + i % nedges; if (dst != i) { + std::vector edgevec; if ((i + dst) % 3 == 1) { - fprintf(f, "%d\t%d\t%d:%d:%d\n", i, dst, i + dst, i + dst + 1, i + dst + 2); + edgevec.push_back(i + dst + 1); + edgevec.push_back(i + dst + 2); } else { - fprintf(f, "%d\t%d\t%d\n", i, dst, i + dst); - + edgevec.push_back(i + dst); + edgevec.push_back(i + dst + 1); } - shouldbe += 2 * (i + dst); + shouldbe += 2 * (i + dst); + + sharderobj.preprocessing_add_edge_multival(i, dst, header_t(i + dst, true), edgevec); } } - } - fclose(f); + } + sharderobj.end_preprocessing(); + + return sharderobj.execute_sharding(get_option_string("nshards", "auto")); + } -class VertexValidator : public VCallback > { +class VertexValidator : public VCallback { public: - virtual void callback(vid_t vertex_id, chivector &vec) { + virtual void callback(vid_t vertex_id, VertexDataType &vec) { int numitems = vertex_id % 10; assert(vec.size() == numitems); @@ -184,9 +203,8 @@ int main(int argc, const char ** argv) { bool scheduler = false; // Whether to use selective scheduling /* Generate data */ - generatedata(filename); - set_conf("filetype", "multivalueedgelist"); - int nshards = convert_if_notexists(filename, "3"); + int nshards = generatedata(filename); + checksum = 0; /* Run */ diff --git a/src/util/kwaymerge.hpp b/src/util/kwaymerge.hpp index a4924a60..6dbea06d 100644 --- a/src/util/kwaymerge.hpp +++ b/src/util/kwaymerge.hpp @@ -41,7 +41,7 @@ template class merge_source { public: virtual bool has_more() = 0; - virtual T next() = 0; + virtual T & next() = 0; }; template class merge_sink { From 2d79519619bc11ff808010fcc30eb7071d575723 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Tue, 13 Aug 2013 22:39:25 -0400 Subject: [PATCH 05/14] write chivector --- src/shards/dynamicdata/dynamicblock.hpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/shards/dynamicdata/dynamicblock.hpp b/src/shards/dynamicdata/dynamicblock.hpp index cb386ce3..4b77d22b 100644 --- a/src/shards/dynamicdata/dynamicblock.hpp +++ b/src/shards/dynamicdata/dynamicblock.hpp @@ -102,17 +102,24 @@ namespace graphchi { // First compute size size = 0; for(int i=0; i < nitems; i++) { - size += chivecs[i].capacity() * sizeof(typename ET::element_type_t) + sizeof(typename ET::sizeword_t); + size += chivecs[i].capacity() * sizeof(typename ET::element_type_t) + sizeof(typename ET::sizeword_t) + sizeof(typename ET::header_t); } *outdata = (uint8_t *) malloc(size); uint8_t * ptr = *outdata; for(int i=0; i < nitems; i++) { ET & vec = chivecs[i]; + + /* Write header */ + ((typename ET::header_t *) ptr)[0] = vec.header(); + ptr += sizeof(typename ET::header_t); + + /* Write size information */ ((uint16_t *) ptr)[0] = vec.size(); ((uint16_t *) ptr)[1] = vec.capacity(); - ptr += sizeof(typename ET::sizeword_t); + + /* Write elements */ vec.write((typename ET::element_type_t *) ptr); ptr += vec.capacity() * sizeof(typename ET::element_type_t); } From 83f83e9f8910ef2350d6912d51f80c510e900119 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Tue, 13 Aug 2013 22:57:03 -0400 Subject: [PATCH 06/14] some memory leak remaining --- src/preprocessing/dynamicdata/sharder.hpp | 7 +++++-- src/tests/test_dynamicedata_loader.cpp | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/preprocessing/dynamicdata/sharder.hpp b/src/preprocessing/dynamicdata/sharder.hpp index 0bef80d3..83197e8d 100644 --- a/src/preprocessing/dynamicdata/sharder.hpp +++ b/src/preprocessing/dynamicdata/sharder.hpp @@ -96,6 +96,7 @@ namespace graphchi { return (dst < x2.dst); } + // TODO: use buffered I/O void reade(int f) { read(f, &src, sizeof(vid_t)); read(f, &dst, sizeof(vid_t)); @@ -191,13 +192,14 @@ namespace graphchi { size_t bufidx; std::vector > buffer; int f; + size_t edgesread; size_t numedges; size_t bufsize_edges; shovel_merge_source(size_t bufsize_bytes, std::string shovelfile) : bufsize_bytes(bufsize_bytes), shovelfile(shovelfile), idx(0), bufidx(0) { f = open(shovelfile.c_str(), O_RDONLY); - + edgesread = 0; if (f < 0) { logstream(LOG_ERROR) << "Could not open shovel file: " << shovelfile << std::endl; printf("Error: %d, %s\n", errno, strerror(errno)); @@ -221,11 +223,12 @@ namespace graphchi { void load_next() { size_t nread = 0; buffer.clear(); - while(nread < bufsize_bytes) { + while(nread < bufsize_bytes && edgesread < numedges) { edge_with_value edge; edge.reade(f); buffer.push_back(edge); nread += sizeof(vid_t) * 2 + sizeof(uint16_t) + sizeof(HeaderDataType) + edge.value.size() * sizeof(VectorElementType); + edgesread++; } bufidx = 0; } diff --git a/src/tests/test_dynamicedata_loader.cpp b/src/tests/test_dynamicedata_loader.cpp index c88edcaf..57405031 100644 --- a/src/tests/test_dynamicedata_loader.cpp +++ b/src/tests/test_dynamicedata_loader.cpp @@ -143,9 +143,9 @@ int generatedata(std::string filename) { std::cout << "Generating data..." << std::endl; shouldbe = 0; - int totalVertices = 200000; // 2 million + int totalVertices = 1000; for(int i=0; i < totalVertices; i++) { - int nedges = random() % 50; + int nedges = random() % 10; for(int j=0; j < nedges; j++) { int dst = (totalVertices / nedges) * j + i % nedges; if (dst != i) { From f5acb84a0db87ff368c4519ec3b9fcf12af95994 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Tue, 13 Aug 2013 23:37:58 -0400 Subject: [PATCH 07/14] temp --- src/preprocessing/dynamicdata/sharder.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/preprocessing/dynamicdata/sharder.hpp b/src/preprocessing/dynamicdata/sharder.hpp index 83197e8d..67352160 100644 --- a/src/preprocessing/dynamicdata/sharder.hpp +++ b/src/preprocessing/dynamicdata/sharder.hpp @@ -105,6 +105,8 @@ namespace graphchi { value.resize(nvalues); read(f, &value[0], sizeof(VectorElementType) * nvalues); read(f, &hdr, sizeof(HeaderDataType)); + + std::cout << src << " - " << dst << " " << *((int*)&value[0]) << std::endl; } void writee(int f) { @@ -209,6 +211,8 @@ namespace graphchi { bufsize_edges = bufsize_bytes / sizeof(edge_with_value); read(f, &numedges, sizeof(size_t)); + + std::cout << "numedges: " << numedges << std::endl; load_next(); } @@ -217,7 +221,6 @@ namespace graphchi { void finish() { close(f); remove(shovelfile.c_str()); - buffer.clear(); } void load_next() { @@ -413,8 +416,8 @@ namespace graphchi { return; } edge_with_value e(from, to, vals, hdr); - e.is_chivec_value = true; + curshovel_buffer[curshovel_idx++] = e; if (curshovel_idx == shovelsize) { flush_shovel(); From fea4349ba8bdef0def2069d4c0146a1e06e692cc Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Wed, 14 Aug 2013 12:11:15 -0400 Subject: [PATCH 08/14] n/a --- .../graphchi_xcode.xcodeproj/akyrola.pbxuser | 105 +++++++++++++++++- 1 file changed, 100 insertions(+), 5 deletions(-) diff --git a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser index 1c856ab8..67be9c87 100644 --- a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser +++ b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser @@ -85,8 +85,8 @@ PBXFileDataSource_Warnings_ColumnID, ); }; - PBXPerProjectTemplateStateSaveDate = 397677050; - PBXWorkspaceStateSaveDate = 397677050; + PBXPerProjectTemplateStateSaveDate = 398189101; + PBXWorkspaceStateSaveDate = 398189101; }; perUserProjectItems = { 5F41F77B17B0057900C5FD90 = 5F41F77B17B0057900C5FD90 /* PBXTextBookmark */; @@ -98,6 +98,14 @@ 5F41F89817B0157600C5FD90 = 5F41F89817B0157600C5FD90 /* PBXTextBookmark */; 5F41F89917B0157600C5FD90 = 5F41F89917B0157600C5FD90 /* PBXTextBookmark */; 5F41F89A17B0157600C5FD90 = 5F41F89A17B0157600C5FD90 /* PBXTextBookmark */; + 5F94518917BBE26C006070F8 /* PBXTextBookmark */ = 5F94518917BBE26C006070F8 /* PBXTextBookmark */; + 5F94518A17BBE26C006070F8 /* PBXBookmark */ = 5F94518A17BBE26C006070F8 /* PBXBookmark */; + 5F94518B17BBE26C006070F8 /* PBXTextBookmark */ = 5F94518B17BBE26C006070F8 /* PBXTextBookmark */; + 5F94519B17BBE2E5006070F8 /* PBXBookmark */ = 5F94519B17BBE2E5006070F8 /* PBXBookmark */; + 5F94519C17BBE389006070F8 /* PBXTextBookmark */ = 5F94519C17BBE389006070F8 /* PBXTextBookmark */; + 5F94519D17BBE389006070F8 /* PBXTextBookmark */ = 5F94519D17BBE389006070F8 /* PBXTextBookmark */; + 5F94519E17BBE389006070F8 /* PBXTextBookmark */ = 5F94519E17BBE389006070F8 /* PBXTextBookmark */; + 5F9451A117BBE389006070F8 /* PBXTextBookmark */ = 5F9451A117BBE389006070F8 /* PBXTextBookmark */; 5FF7741517B170E300F84939 = 5FF7741517B170E300F84939 /* PBXBookmark */; 5FF7741617B170E300F84939 = 5FF7741617B170E300F84939 /* PBXTextBookmark */; }; @@ -196,6 +204,11 @@ name = kwaymerge.hpp; path = ../src/util/kwaymerge.hpp; sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {950, 1469}}"; + sepNavSelRange = "{2572, 0}"; + sepNavVisRange = "{2062, 561}"; + }; }; 5F41F7B417B0073000C5FD90 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -288,6 +301,13 @@ vrLen = 2451; vrLoc = 1099; }; + 5F54B84615FD4B2500B3842C /* chivector.hpp */ = { + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {964, 1807}}"; + sepNavSelRange = "{0, 0}"; + sepNavVisRange = "{0, 1270}"; + }; + }; 5F54B84915FD4D9F00B3842C /* test_dynamicedata */ = { activeExec = 0; executables = ( @@ -408,6 +428,11 @@ name = sharder.hpp; path = ../src/preprocessing/sharder.hpp; sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1181, 13364}}"; + sepNavSelRange = "{22125, 156}"; + sepNavVisRange = "{20753, 2847}"; + }; }; 5F7A0FE11589160900748D0D /* Source Control */ = { isa = PBXSourceControlManager; @@ -595,6 +620,74 @@ 5FCC20ED159DE4660003D0E9 /* connectedcomponents */, ); }; + 5F94518917BBE26C006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F54B84615FD4B2500B3842C /* chivector.hpp */; + name = "chivector.hpp: 1"; + rLen = 0; + rLoc = 0; + rType = 0; + vrLen = 1270; + vrLoc = 0; + }; + 5F94518A17BBE26C006070F8 /* PBXBookmark */ = { + isa = PBXBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + }; + 5F94518B17BBE26C006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 31"; + rLen = 0; + rLoc = 950; + rType = 0; + vrLen = 2068; + vrLoc = 3484; + }; + 5F94519B17BBE2E5006070F8 /* PBXBookmark */ = { + isa = PBXBookmark; + fRef = 5FCC1CD81594F1890003D0E9 /* sharder.hpp */; + }; + 5F94519C17BBE389006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F78366C15DD718A00954509 /* sharder.hpp */; + name = "sharder.hpp: 1"; + rLen = 0; + rLoc = 0; + rType = 0; + vrLen = 1899; + vrLoc = 0; + }; + 5F94519D17BBE389006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 671"; + rLen = 42; + rLoc = 25588; + rType = 0; + vrLen = 3081; + vrLoc = 25228; + }; + 5F94519E17BBE389006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 708"; + rLen = 0; + rLoc = 27339; + rType = 0; + vrLen = 3115; + vrLoc = 25894; + }; + 5F9451A117BBE389006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F78366C15DD718A00954509 /* sharder.hpp */; + name = "sharder.hpp: 622"; + rLen = 156; + rLoc = 22125; + rType = 0; + vrLen = 2847; + vrLoc = 20753; + }; 5F9EBAAC1593F87D0092F7E2 /* chifilenames.hpp */ = { uiCtxt = { sepNavIntBoundsRect = "{{0, 0}, {1076, 5681}}"; @@ -667,6 +760,8 @@ sourceDirectories = ( ); startupPath = "/Users/akyrola/Projects/GraphCHI/graphchi-cpp/"; + variableFormatDictionary = { + }; }; 5FCB4691174AA11C0023D9D7 /* gensgd */ = { isa = PBXExecutable; @@ -1248,9 +1343,9 @@ }; 5FF7749917B3E1A200F84939 /* sharder.hpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {964, 15288}}"; - sepNavSelRange = "{950, 0}"; - sepNavVisRange = "{750, 1071}"; + sepNavIntBoundsRect = "{{0, 0}, {1272, 10764}}"; + sepNavSelRange = "{27339, 0}"; + sepNavVisRange = "{25894, 3115}"; }; }; 8DD76F620486A84900D96B5E /* pagerank */ = { From 5e594f2ac316f5bd13acc2049a7d692c9e3851c1 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Wed, 14 Aug 2013 12:16:23 -0400 Subject: [PATCH 09/14] merged --- src/preprocessing/conversions.hpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/preprocessing/conversions.hpp b/src/preprocessing/conversions.hpp index 45d13eb3..b643d053 100644 --- a/src/preprocessing/conversions.hpp +++ b/src/preprocessing/conversions.hpp @@ -654,14 +654,9 @@ namespace graphchi { * Converts a graph input to shards with no edge values. Preprocessing has several steps, * see sharder.hpp for more information. */ -<<<<<<< HEAD - int convert_none(std::string basefilename, std::string nshards_string); - int convert_none(std::string basefilename, std::string nshards_string) { - -======= + static int VARIABLE_IS_NOT_USED convert_none(std::string basefilename, std::string nshards_string); static int VARIABLE_IS_NOT_USED convert_none(std::string basefilename, std::string nshards_string) { ->>>>>>> 13bfbb4b4436505ef2f94cc03edb6bd689bff699 sharder sharderobj(basefilename); sharderobj.set_no_edgevalues(); From 3109a1cc5a26cddd2b8a79614b4e479ed715bf7f Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Wed, 14 Aug 2013 13:01:56 -0400 Subject: [PATCH 10/14] fixed size calculations --- .../graphchi_xcode.xcodeproj/akyrola.pbxuser | 1143 +++++++++++++---- .../dynamicdata/vertex_data_dynamic.hpp | 7 +- src/preprocessing/dynamicdata/sharder.hpp | 62 +- src/util/kwaymerge.hpp | 2 +- 4 files changed, 912 insertions(+), 302 deletions(-) diff --git a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser index d0203994..18107a65 100644 --- a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser +++ b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser @@ -2,14 +2,17 @@ { 08FB7793FE84155DC02AAC07 /* Project object */ = { activeBuildConfigurationName = Debug; - activeExecutable = 5F648EED17B82B9900EFEE76 /* SCC */; - activeTarget = 5F648EE317B82B9900EFEE76 /* SCC */; + activeExecutable = 5FCB4690174AA11C0023D9D7 /* test_dynamicedata */; + activeTarget = 5F54B84915FD4D9F00B3842C /* test_dynamicedata */; addToTargets = ( 5F8DD040159DDABC0034C889 /* connectedcomponents */, ); breakpoints = ( 5FCC250915A392C70003D0E9 /* streaming_pagerank.cpp:267 */, 5F648F0617B8332100EFEE76 /* stronglyconnectedcomponents.cpp:236 */, + 5F9451B217BBE4CF006070F8 /* conversions.hpp:264 */, + 5F9451E417BBE83D006070F8 /* sharder.hpp:753 */, + 5F9451E917BBE8D2006070F8 /* sharder.hpp:451 */, ); codeSenseManager = 5F7A0FE21589160900748D0D /* Code sense */; executables = ( @@ -49,7 +52,7 @@ PBXFileTableDataSourceColumnSortingKey = PBXFileDataSource_Filename_ColumnID; PBXFileTableDataSourceColumnWidthsKey = ( 20, - 786, + 1049, 20, 48, 43, @@ -88,8 +91,8 @@ PBXFileDataSource_Warnings_ColumnID, ); }; - PBXPerProjectTemplateStateSaveDate = 397938024; - PBXWorkspaceStateSaveDate = 397938024; + PBXPerProjectTemplateStateSaveDate = 398189638; + PBXWorkspaceStateSaveDate = 398189638; }; perUserProjectItems = { 5F41F77B17B0057900C5FD90 = 5F41F77B17B0057900C5FD90 /* PBXTextBookmark */; @@ -119,205 +122,252 @@ 5F41F8C317B0170500C5FD90 = 5F41F8C317B0170500C5FD90 /* PBXTextBookmark */; 5F41F8C417B0170500C5FD90 = 5F41F8C417B0170500C5FD90 /* PBXTextBookmark */; 5F41F8C517B0170500C5FD90 = 5F41F8C517B0170500C5FD90 /* PBXTextBookmark */; - 5F648EE017B82ABB00EFEE76 /* PBXBookmark */ = 5F648EE017B82ABB00EFEE76 /* PBXBookmark */; - 5F648F0917B8337200EFEE76 /* PBXTextBookmark */ = 5F648F0917B8337200EFEE76 /* PBXTextBookmark */; - 5F648F0A17B8337200EFEE76 /* XCBuildMessageTextBookmark */ = 5F648F0A17B8337200EFEE76 /* XCBuildMessageTextBookmark */; - 5F648F0C17B8337200EFEE76 /* PBXTextBookmark */ = 5F648F0C17B8337200EFEE76 /* PBXTextBookmark */; - 5F648F2517B8340900EFEE76 /* PBXTextBookmark */ = 5F648F2517B8340900EFEE76 /* PBXTextBookmark */; - 5F648F2617B8340900EFEE76 /* PBXTextBookmark */ = 5F648F2617B8340900EFEE76 /* PBXTextBookmark */; - 5F648F2717B8340900EFEE76 /* XCBuildMessageTextBookmark */ = 5F648F2717B8340900EFEE76 /* XCBuildMessageTextBookmark */; - 5F648F2917B8340900EFEE76 /* PBXTextBookmark */ = 5F648F2917B8340900EFEE76 /* PBXTextBookmark */; - 5F648F2A17B8340900EFEE76 /* PBXTextBookmark */ = 5F648F2A17B8340900EFEE76 /* PBXTextBookmark */; - 5F648F2B17B8340900EFEE76 /* PBXTextBookmark */ = 5F648F2B17B8340900EFEE76 /* PBXTextBookmark */; - 5F648F3017B8340900EFEE76 /* PBXTextBookmark */ = 5F648F3017B8340900EFEE76 /* PBXTextBookmark */; - 5F648F4117B8352800EFEE76 /* PBXTextBookmark */ = 5F648F4117B8352800EFEE76 /* PBXTextBookmark */; - 5F648F4217B8352800EFEE76 /* PBXTextBookmark */ = 5F648F4217B8352800EFEE76 /* PBXTextBookmark */; - 5F648F4617B8358B00EFEE76 /* PBXTextBookmark */ = 5F648F4617B8358B00EFEE76 /* PBXTextBookmark */; - 5F648F4717B8358B00EFEE76 /* PBXTextBookmark */ = 5F648F4717B8358B00EFEE76 /* PBXTextBookmark */; - 5F648F4E17B835DD00EFEE76 /* XCBuildMessageTextBookmark */ = 5F648F4E17B835DD00EFEE76 /* XCBuildMessageTextBookmark */; - 5F648F5017B835DD00EFEE76 /* PBXTextBookmark */ = 5F648F5017B835DD00EFEE76 /* PBXTextBookmark */; - 5F648F5117B835DD00EFEE76 /* PBXTextBookmark */ = 5F648F5117B835DD00EFEE76 /* PBXTextBookmark */; - 5F648F5217B835DD00EFEE76 /* PBXTextBookmark */ = 5F648F5217B835DD00EFEE76 /* PBXTextBookmark */; - 5F648F7417B839AD00EFEE76 /* PBXBookmark */ = 5F648F7417B839AD00EFEE76 /* PBXBookmark */; - 5F648F7F17B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F7F17B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8017B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8017B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8117B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8117B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8217B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8217B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8317B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8317B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8417B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8417B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8517B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8517B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8617B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8617B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8717B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8717B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8817B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8817B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8917B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8917B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8A17B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8A17B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8B17B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8B17B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8E17B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8E17B83A6500EFEE76 /* PBXTextBookmark */; - 5F648F8F17B83A6500EFEE76 /* PBXTextBookmark */ = 5F648F8F17B83A6500EFEE76 /* PBXTextBookmark */; - 5F648FB517B83E0E00EFEE76 /* PBXTextBookmark */ = 5F648FB517B83E0E00EFEE76 /* PBXTextBookmark */; - 5F648FB617B83E0E00EFEE76 /* PBXTextBookmark */ = 5F648FB617B83E0E00EFEE76 /* PBXTextBookmark */; - 5F648FB717B83E0E00EFEE76 /* PBXTextBookmark */ = 5F648FB717B83E0E00EFEE76 /* PBXTextBookmark */; - 5F648FB817B83E0E00EFEE76 /* PBXTextBookmark */ = 5F648FB817B83E0E00EFEE76 /* PBXTextBookmark */; - 5F648FB917B83E0E00EFEE76 /* PBXTextBookmark */ = 5F648FB917B83E0E00EFEE76 /* PBXTextBookmark */; - 5F648FBD17B83EF200EFEE76 /* PBXTextBookmark */ = 5F648FBD17B83EF200EFEE76 /* PBXTextBookmark */; - 5F648FBE17B83EF200EFEE76 /* PBXTextBookmark */ = 5F648FBE17B83EF200EFEE76 /* PBXTextBookmark */; - 5F648FBF17B83EF200EFEE76 /* PBXTextBookmark */ = 5F648FBF17B83EF200EFEE76 /* PBXTextBookmark */; - 5F648FCD17B94D1C00EFEE76 /* PBXBookmark */ = 5F648FCD17B94D1C00EFEE76 /* PBXBookmark */; - 5F648FD117B94E7100EFEE76 /* PBXBookmark */ = 5F648FD117B94E7100EFEE76 /* PBXBookmark */; - 5F648FED17B9634C00EFEE76 /* PBXBookmark */ = 5F648FED17B9634C00EFEE76 /* PBXBookmark */; - 5F648FFE17B9651E00EFEE76 /* PBXBookmark */ = 5F648FFE17B9651E00EFEE76 /* PBXBookmark */; - 5F64900817B9691F00EFEE76 /* PBXBookmark */ = 5F64900817B9691F00EFEE76 /* PBXBookmark */; - 5F64901317B969F000EFEE76 /* PBXTextBookmark */ = 5F64901317B969F000EFEE76 /* PBXTextBookmark */; - 5F64901417B969F000EFEE76 /* PBXTextBookmark */ = 5F64901417B969F000EFEE76 /* PBXTextBookmark */; - 5F64901517B969F000EFEE76 /* PBXTextBookmark */ = 5F64901517B969F000EFEE76 /* PBXTextBookmark */; - 5F64901617B969F000EFEE76 /* PBXTextBookmark */ = 5F64901617B969F000EFEE76 /* PBXTextBookmark */; - 5F64901717B969F000EFEE76 /* PBXTextBookmark */ = 5F64901717B969F000EFEE76 /* PBXTextBookmark */; - 5F64901817B969F000EFEE76 /* PBXTextBookmark */ = 5F64901817B969F000EFEE76 /* PBXTextBookmark */; - 5F64901917B969F000EFEE76 /* PBXTextBookmark */ = 5F64901917B969F000EFEE76 /* PBXTextBookmark */; - 5F64901A17B969F000EFEE76 /* PBXTextBookmark */ = 5F64901A17B969F000EFEE76 /* PBXTextBookmark */; - 5F64901B17B969F000EFEE76 /* PBXTextBookmark */ = 5F64901B17B969F000EFEE76 /* PBXTextBookmark */; - 5F64901C17B969F000EFEE76 /* PBXTextBookmark */ = 5F64901C17B969F000EFEE76 /* PBXTextBookmark */; - 5F64901D17B969F000EFEE76 /* PBXTextBookmark */ = 5F64901D17B969F000EFEE76 /* PBXTextBookmark */; - 5F64901E17B969F000EFEE76 /* PBXTextBookmark */ = 5F64901E17B969F000EFEE76 /* PBXTextBookmark */; - 5F64901F17B969F000EFEE76 /* PBXTextBookmark */ = 5F64901F17B969F000EFEE76 /* PBXTextBookmark */; - 5F64902017B969F000EFEE76 /* PBXTextBookmark */ = 5F64902017B969F000EFEE76 /* PBXTextBookmark */; - 5F64902317B969F000EFEE76 /* PBXTextBookmark */ = 5F64902317B969F000EFEE76 /* PBXTextBookmark */; - 5F64902617B969F000EFEE76 /* PBXTextBookmark */ = 5F64902617B969F000EFEE76 /* PBXTextBookmark */; - 5F64902717B969F000EFEE76 /* PBXTextBookmark */ = 5F64902717B969F000EFEE76 /* PBXTextBookmark */; - 5F64902A17B969F000EFEE76 /* PBXTextBookmark */ = 5F64902A17B969F000EFEE76 /* PBXTextBookmark */; - 5F64902D17B969F000EFEE76 /* PBXTextBookmark */ = 5F64902D17B969F000EFEE76 /* PBXTextBookmark */; - 5F64902E17B969F000EFEE76 /* PBXTextBookmark */ = 5F64902E17B969F000EFEE76 /* PBXTextBookmark */; - 5F64902F17B969F000EFEE76 /* PBXTextBookmark */ = 5F64902F17B969F000EFEE76 /* PBXTextBookmark */; - 5F64903017B969F000EFEE76 /* PBXTextBookmark */ = 5F64903017B969F000EFEE76 /* PBXTextBookmark */; - 5F64903317B969F000EFEE76 /* PBXTextBookmark */ = 5F64903317B969F000EFEE76 /* PBXTextBookmark */; - 5F64903617B969F000EFEE76 /* PBXTextBookmark */ = 5F64903617B969F000EFEE76 /* PBXTextBookmark */; - 5F64903917B969F000EFEE76 /* PBXTextBookmark */ = 5F64903917B969F000EFEE76 /* PBXTextBookmark */; - 5F64903A17B969F000EFEE76 /* PBXTextBookmark */ = 5F64903A17B969F000EFEE76 /* PBXTextBookmark */; - 5F64903B17B96A2700EFEE76 /* PBXTextBookmark */ = 5F64903B17B96A2700EFEE76 /* PBXTextBookmark */; - 5F64903C17B96A2700EFEE76 /* PBXTextBookmark */ = 5F64903C17B96A2700EFEE76 /* PBXTextBookmark */; - 5F64903D17B96A2700EFEE76 /* PBXTextBookmark */ = 5F64903D17B96A2700EFEE76 /* PBXTextBookmark */; - 5F64903E17B96A2700EFEE76 /* PBXTextBookmark */ = 5F64903E17B96A2700EFEE76 /* PBXTextBookmark */; - 5F64903F17B96A2700EFEE76 /* PBXTextBookmark */ = 5F64903F17B96A2700EFEE76 /* PBXTextBookmark */; - 5F64904017B96A2700EFEE76 /* PBXTextBookmark */ = 5F64904017B96A2700EFEE76 /* PBXTextBookmark */; - 5F64904117B96A2700EFEE76 /* PBXTextBookmark */ = 5F64904117B96A2700EFEE76 /* PBXTextBookmark */; - 5F64904217B96A2700EFEE76 /* PBXTextBookmark */ = 5F64904217B96A2700EFEE76 /* PBXTextBookmark */; - 5F64904317B96A2700EFEE76 /* PBXTextBookmark */ = 5F64904317B96A2700EFEE76 /* PBXTextBookmark */; - 5F64904417B96A2700EFEE76 /* PBXTextBookmark */ = 5F64904417B96A2700EFEE76 /* PBXTextBookmark */; - 5F64904D17B96AA300EFEE76 /* PBXTextBookmark */ = 5F64904D17B96AA300EFEE76 /* PBXTextBookmark */; - 5F64904E17B96AA300EFEE76 /* PBXTextBookmark */ = 5F64904E17B96AA300EFEE76 /* PBXTextBookmark */; - 5F64904F17B96AA300EFEE76 /* PBXTextBookmark */ = 5F64904F17B96AA300EFEE76 /* PBXTextBookmark */; - 5F64905017B96AA300EFEE76 /* PBXTextBookmark */ = 5F64905017B96AA300EFEE76 /* PBXTextBookmark */; - 5F64905117B96AA300EFEE76 /* PBXTextBookmark */ = 5F64905117B96AA300EFEE76 /* PBXTextBookmark */; - 5F64905217B96AA300EFEE76 /* PBXTextBookmark */ = 5F64905217B96AA300EFEE76 /* PBXTextBookmark */; - 5F64905317B96AA300EFEE76 /* PBXTextBookmark */ = 5F64905317B96AA300EFEE76 /* PBXTextBookmark */; - 5F64905417B96AA300EFEE76 /* PBXTextBookmark */ = 5F64905417B96AA300EFEE76 /* PBXTextBookmark */; - 5F64905517B96AA300EFEE76 /* PBXTextBookmark */ = 5F64905517B96AA300EFEE76 /* PBXTextBookmark */; - 5F64905617B96AA300EFEE76 /* PBXTextBookmark */ = 5F64905617B96AA300EFEE76 /* PBXTextBookmark */; - 5F64905717B96AA300EFEE76 /* PBXTextBookmark */ = 5F64905717B96AA300EFEE76 /* PBXTextBookmark */; - 5F64905E17B96AEB00EFEE76 /* PBXTextBookmark */ = 5F64905E17B96AEB00EFEE76 /* PBXTextBookmark */; - 5F64905F17B96AEB00EFEE76 /* PBXTextBookmark */ = 5F64905F17B96AEB00EFEE76 /* PBXTextBookmark */; - 5F64906017B96AEB00EFEE76 /* PBXTextBookmark */ = 5F64906017B96AEB00EFEE76 /* PBXTextBookmark */; - 5F64906117B96AEB00EFEE76 /* PBXTextBookmark */ = 5F64906117B96AEB00EFEE76 /* PBXTextBookmark */; - 5F64906317B96AEB00EFEE76 /* XCBuildMessageTextBookmark */ = 5F64906317B96AEB00EFEE76 /* XCBuildMessageTextBookmark */; - 5F64906517B96AEB00EFEE76 /* PBXTextBookmark */ = 5F64906517B96AEB00EFEE76 /* PBXTextBookmark */; - 5F64906A17B96BB800EFEE76 /* PBXTextBookmark */ = 5F64906A17B96BB800EFEE76 /* PBXTextBookmark */; - 5F64906B17B96BB800EFEE76 /* PBXTextBookmark */ = 5F64906B17B96BB800EFEE76 /* PBXTextBookmark */; - 5F64906C17B96BB800EFEE76 /* PBXTextBookmark */ = 5F64906C17B96BB800EFEE76 /* PBXTextBookmark */; - 5F64906D17B96BB800EFEE76 /* PBXTextBookmark */ = 5F64906D17B96BB800EFEE76 /* PBXTextBookmark */; - 5F64906E17B96BB800EFEE76 /* PBXTextBookmark */ = 5F64906E17B96BB800EFEE76 /* PBXTextBookmark */; - 5F64906F17B96BB800EFEE76 /* PBXTextBookmark */ = 5F64906F17B96BB800EFEE76 /* PBXTextBookmark */; - 5F64907017B96BB800EFEE76 /* PBXTextBookmark */ = 5F64907017B96BB800EFEE76 /* PBXTextBookmark */; - 5F64907117B96BB800EFEE76 /* PBXTextBookmark */ = 5F64907117B96BB800EFEE76 /* PBXTextBookmark */; - 5F64907217B96BB800EFEE76 /* PBXTextBookmark */ = 5F64907217B96BB800EFEE76 /* PBXTextBookmark */; - 5F64907317B96BB800EFEE76 /* PBXTextBookmark */ = 5F64907317B96BB800EFEE76 /* PBXTextBookmark */; - 5F64907417B96BB800EFEE76 /* PBXTextBookmark */ = 5F64907417B96BB800EFEE76 /* PBXTextBookmark */; - 5F64907517B96BB800EFEE76 /* PBXTextBookmark */ = 5F64907517B96BB800EFEE76 /* PBXTextBookmark */; - 5F64907F17B96C1100EFEE76 /* PBXTextBookmark */ = 5F64907F17B96C1100EFEE76 /* PBXTextBookmark */; - 5F64908017B96C1100EFEE76 /* XCBuildMessageTextBookmark */ = 5F64908017B96C1100EFEE76 /* XCBuildMessageTextBookmark */; - 5F64908217B96C1100EFEE76 /* PBXTextBookmark */ = 5F64908217B96C1100EFEE76 /* PBXTextBookmark */; - 5F64908517B96C1A00EFEE76 /* PBXTextBookmark */ = 5F64908517B96C1A00EFEE76 /* PBXTextBookmark */; - 5F64908617B96C1A00EFEE76 /* PBXTextBookmark */ = 5F64908617B96C1A00EFEE76 /* PBXTextBookmark */; - 5F64908717B96C1A00EFEE76 /* PBXTextBookmark */ = 5F64908717B96C1A00EFEE76 /* PBXTextBookmark */; - 5F64908817B96C1A00EFEE76 /* PBXTextBookmark */ = 5F64908817B96C1A00EFEE76 /* PBXTextBookmark */; - 5F64908917B96C1A00EFEE76 /* PBXTextBookmark */ = 5F64908917B96C1A00EFEE76 /* PBXTextBookmark */; - 5F64908A17B96C1A00EFEE76 /* PBXTextBookmark */ = 5F64908A17B96C1A00EFEE76 /* PBXTextBookmark */; - 5F64908B17B96C1A00EFEE76 /* PBXTextBookmark */ = 5F64908B17B96C1A00EFEE76 /* PBXTextBookmark */; - 5F64908C17B96C1A00EFEE76 /* PBXTextBookmark */ = 5F64908C17B96C1A00EFEE76 /* PBXTextBookmark */; - 5F64908D17B96C1A00EFEE76 /* PBXTextBookmark */ = 5F64908D17B96C1A00EFEE76 /* PBXTextBookmark */; - 5F64908E17B96C1A00EFEE76 /* PBXTextBookmark */ = 5F64908E17B96C1A00EFEE76 /* PBXTextBookmark */; - 5F64908F17B9743100EFEE76 /* PBXTextBookmark */ = 5F64908F17B9743100EFEE76 /* PBXTextBookmark */; - 5F64909017B9743100EFEE76 /* PBXTextBookmark */ = 5F64909017B9743100EFEE76 /* PBXTextBookmark */; - 5F64909117B9743100EFEE76 /* PBXTextBookmark */ = 5F64909117B9743100EFEE76 /* PBXTextBookmark */; - 5F64909217B9743100EFEE76 /* PBXTextBookmark */ = 5F64909217B9743100EFEE76 /* PBXTextBookmark */; - 5F64909317B9743100EFEE76 /* PBXTextBookmark */ = 5F64909317B9743100EFEE76 /* PBXTextBookmark */; - 5F64909417B9743100EFEE76 /* PBXTextBookmark */ = 5F64909417B9743100EFEE76 /* PBXTextBookmark */; - 5F64909517B9743100EFEE76 /* PBXTextBookmark */ = 5F64909517B9743100EFEE76 /* PBXTextBookmark */; - 5F64909617B9743100EFEE76 /* PBXTextBookmark */ = 5F64909617B9743100EFEE76 /* PBXTextBookmark */; - 5F64909717B9743100EFEE76 /* PBXTextBookmark */ = 5F64909717B9743100EFEE76 /* PBXTextBookmark */; - 5F64909817B9743100EFEE76 /* PBXTextBookmark */ = 5F64909817B9743100EFEE76 /* PBXTextBookmark */; - 5F64909917B9743100EFEE76 /* PBXTextBookmark */ = 5F64909917B9743100EFEE76 /* PBXTextBookmark */; - 5F64909A17B9743100EFEE76 /* PBXTextBookmark */ = 5F64909A17B9743100EFEE76 /* PBXTextBookmark */; - 5F64909B17B9743100EFEE76 /* PBXTextBookmark */ = 5F64909B17B9743100EFEE76 /* PBXTextBookmark */; - 5F64909C17B9743100EFEE76 /* PBXTextBookmark */ = 5F64909C17B9743100EFEE76 /* PBXTextBookmark */; - 5F6490A117B9748500EFEE76 /* PBXTextBookmark */ = 5F6490A117B9748500EFEE76 /* PBXTextBookmark */; - 5F6490A217B9748500EFEE76 /* PBXTextBookmark */ = 5F6490A217B9748500EFEE76 /* PBXTextBookmark */; - 5F6490A317B9748500EFEE76 /* PBXTextBookmark */ = 5F6490A317B9748500EFEE76 /* PBXTextBookmark */; - 5F6490A417B9748500EFEE76 /* PBXTextBookmark */ = 5F6490A417B9748500EFEE76 /* PBXTextBookmark */; - 5F6490A517B9748500EFEE76 /* PBXTextBookmark */ = 5F6490A517B9748500EFEE76 /* PBXTextBookmark */; - 5F6490A617B9748500EFEE76 /* PBXTextBookmark */ = 5F6490A617B9748500EFEE76 /* PBXTextBookmark */; - 5F6490A717B9748500EFEE76 /* PBXTextBookmark */ = 5F6490A717B9748500EFEE76 /* PBXTextBookmark */; - 5F6490A817B9748500EFEE76 /* PBXTextBookmark */ = 5F6490A817B9748500EFEE76 /* PBXTextBookmark */; - 5F6490A917B9748500EFEE76 /* PBXTextBookmark */ = 5F6490A917B9748500EFEE76 /* PBXTextBookmark */; - 5F6490AA17B9748500EFEE76 /* PBXTextBookmark */ = 5F6490AA17B9748500EFEE76 /* PBXTextBookmark */; - 5F6490AB17B9748500EFEE76 /* PBXTextBookmark */ = 5F6490AB17B9748500EFEE76 /* PBXTextBookmark */; - 5F6490AC17B9748500EFEE76 /* PBXTextBookmark */ = 5F6490AC17B9748500EFEE76 /* PBXTextBookmark */; - 5F6490AD17B974B200EFEE76 /* PBXTextBookmark */ = 5F6490AD17B974B200EFEE76 /* PBXTextBookmark */; - 5F6490AE17B974B200EFEE76 /* PBXTextBookmark */ = 5F6490AE17B974B200EFEE76 /* PBXTextBookmark */; - 5F6490AF17B974B200EFEE76 /* PBXTextBookmark */ = 5F6490AF17B974B200EFEE76 /* PBXTextBookmark */; - 5F6490B017B974B200EFEE76 /* PBXTextBookmark */ = 5F6490B017B974B200EFEE76 /* PBXTextBookmark */; - 5F6490B117B974B200EFEE76 /* PBXTextBookmark */ = 5F6490B117B974B200EFEE76 /* PBXTextBookmark */; - 5F6490B217B974B200EFEE76 /* PBXTextBookmark */ = 5F6490B217B974B200EFEE76 /* PBXTextBookmark */; - 5F6490B317B974B200EFEE76 /* PBXTextBookmark */ = 5F6490B317B974B200EFEE76 /* PBXTextBookmark */; - 5F6490B417B974B200EFEE76 /* PBXTextBookmark */ = 5F6490B417B974B200EFEE76 /* PBXTextBookmark */; - 5F6490B517B974B200EFEE76 /* PBXTextBookmark */ = 5F6490B517B974B200EFEE76 /* PBXTextBookmark */; - 5F6490B617B974B200EFEE76 /* PBXTextBookmark */ = 5F6490B617B974B200EFEE76 /* PBXTextBookmark */; - 5F6490B717B974B200EFEE76 /* PBXTextBookmark */ = 5F6490B717B974B200EFEE76 /* PBXTextBookmark */; - 5F6490B817B974B200EFEE76 /* PBXTextBookmark */ = 5F6490B817B974B200EFEE76 /* PBXTextBookmark */; - 5F6490B917B974C000EFEE76 /* PBXTextBookmark */ = 5F6490B917B974C000EFEE76 /* PBXTextBookmark */; - 5F6490BA17B974C000EFEE76 /* PBXTextBookmark */ = 5F6490BA17B974C000EFEE76 /* PBXTextBookmark */; - 5F6490BB17B974C000EFEE76 /* PBXTextBookmark */ = 5F6490BB17B974C000EFEE76 /* PBXTextBookmark */; - 5F6490BC17B974C000EFEE76 /* PBXTextBookmark */ = 5F6490BC17B974C000EFEE76 /* PBXTextBookmark */; - 5F6490BD17B974C000EFEE76 /* PBXTextBookmark */ = 5F6490BD17B974C000EFEE76 /* PBXTextBookmark */; - 5F6490BE17B974C000EFEE76 /* PBXTextBookmark */ = 5F6490BE17B974C000EFEE76 /* PBXTextBookmark */; - 5F6490BF17B974C000EFEE76 /* PBXTextBookmark */ = 5F6490BF17B974C000EFEE76 /* PBXTextBookmark */; - 5F6490C017B974C000EFEE76 /* PBXTextBookmark */ = 5F6490C017B974C000EFEE76 /* PBXTextBookmark */; - 5F6490C117B974C000EFEE76 /* PBXTextBookmark */ = 5F6490C117B974C000EFEE76 /* PBXTextBookmark */; - 5F6490C217B974C000EFEE76 /* PBXTextBookmark */ = 5F6490C217B974C000EFEE76 /* PBXTextBookmark */; - 5F6490C317B974C000EFEE76 /* PBXTextBookmark */ = 5F6490C317B974C000EFEE76 /* PBXTextBookmark */; - 5F6490C417B974C000EFEE76 /* PBXTextBookmark */ = 5F6490C417B974C000EFEE76 /* PBXTextBookmark */; - 5F6490C517BA91DE00EFEE76 /* PBXTextBookmark */ = 5F6490C517BA91DE00EFEE76 /* PBXTextBookmark */; - 5F6490C617BA91DE00EFEE76 /* PBXTextBookmark */ = 5F6490C617BA91DE00EFEE76 /* PBXTextBookmark */; - 5F6490C717BA91DE00EFEE76 /* PBXTextBookmark */ = 5F6490C717BA91DE00EFEE76 /* PBXTextBookmark */; - 5F6490C817BA91DE00EFEE76 /* PBXTextBookmark */ = 5F6490C817BA91DE00EFEE76 /* PBXTextBookmark */; - 5F6490C917BA91DE00EFEE76 /* PBXTextBookmark */ = 5F6490C917BA91DE00EFEE76 /* PBXTextBookmark */; - 5F6490CA17BA91DE00EFEE76 /* PBXTextBookmark */ = 5F6490CA17BA91DE00EFEE76 /* PBXTextBookmark */; - 5F6490CB17BA91DE00EFEE76 /* PBXTextBookmark */ = 5F6490CB17BA91DE00EFEE76 /* PBXTextBookmark */; - 5F6490CC17BA91DE00EFEE76 /* PBXTextBookmark */ = 5F6490CC17BA91DE00EFEE76 /* PBXTextBookmark */; - 5F6490CD17BA91DE00EFEE76 /* PBXTextBookmark */ = 5F6490CD17BA91DE00EFEE76 /* PBXTextBookmark */; - 5F6490CE17BA91DE00EFEE76 /* PBXTextBookmark */ = 5F6490CE17BA91DE00EFEE76 /* PBXTextBookmark */; - 5F6490CF17BA91DE00EFEE76 /* PBXTextBookmark */ = 5F6490CF17BA91DE00EFEE76 /* PBXTextBookmark */; - 5F6490D017BA91DE00EFEE76 /* PBXTextBookmark */ = 5F6490D017BA91DE00EFEE76 /* PBXTextBookmark */; - 5F6490D117BA966600EFEE76 /* PBXTextBookmark */ = 5F6490D117BA966600EFEE76 /* PBXTextBookmark */; - 5F6490D217BA966600EFEE76 /* PBXTextBookmark */ = 5F6490D217BA966600EFEE76 /* PBXTextBookmark */; - 5F6490D317BA966600EFEE76 /* PBXTextBookmark */ = 5F6490D317BA966600EFEE76 /* PBXTextBookmark */; - 5F6490D417BA966600EFEE76 /* PBXTextBookmark */ = 5F6490D417BA966600EFEE76 /* PBXTextBookmark */; - 5F6490D517BA966600EFEE76 /* PBXTextBookmark */ = 5F6490D517BA966600EFEE76 /* PBXTextBookmark */; - 5F6490D617BA966600EFEE76 /* PBXTextBookmark */ = 5F6490D617BA966600EFEE76 /* PBXTextBookmark */; - 5F6490D717BA966600EFEE76 /* PBXTextBookmark */ = 5F6490D717BA966600EFEE76 /* PBXTextBookmark */; - 5F6490D817BA966600EFEE76 /* PBXTextBookmark */ = 5F6490D817BA966600EFEE76 /* PBXTextBookmark */; - 5F6490D917BA966600EFEE76 /* PBXTextBookmark */ = 5F6490D917BA966600EFEE76 /* PBXTextBookmark */; - 5F6490DA17BA966600EFEE76 /* PBXTextBookmark */ = 5F6490DA17BA966600EFEE76 /* PBXTextBookmark */; - 5F6490DB17BA966600EFEE76 /* PBXTextBookmark */ = 5F6490DB17BA966600EFEE76 /* PBXTextBookmark */; + 5F648EE017B82ABB00EFEE76 = 5F648EE017B82ABB00EFEE76 /* PBXBookmark */; + 5F648F0917B8337200EFEE76 = 5F648F0917B8337200EFEE76 /* PBXTextBookmark */; + 5F648F0A17B8337200EFEE76 = 5F648F0A17B8337200EFEE76 /* PBXTextBookmark */; + 5F648F0C17B8337200EFEE76 = 5F648F0C17B8337200EFEE76 /* PBXTextBookmark */; + 5F648F2517B8340900EFEE76 = 5F648F2517B8340900EFEE76 /* PBXTextBookmark */; + 5F648F2617B8340900EFEE76 = 5F648F2617B8340900EFEE76 /* PBXTextBookmark */; + 5F648F2717B8340900EFEE76 = 5F648F2717B8340900EFEE76 /* PBXTextBookmark */; + 5F648F2917B8340900EFEE76 = 5F648F2917B8340900EFEE76 /* PBXTextBookmark */; + 5F648F2A17B8340900EFEE76 = 5F648F2A17B8340900EFEE76 /* PBXTextBookmark */; + 5F648F2B17B8340900EFEE76 = 5F648F2B17B8340900EFEE76 /* PBXTextBookmark */; + 5F648F3017B8340900EFEE76 = 5F648F3017B8340900EFEE76 /* PBXTextBookmark */; + 5F648F4117B8352800EFEE76 = 5F648F4117B8352800EFEE76 /* PBXTextBookmark */; + 5F648F4217B8352800EFEE76 = 5F648F4217B8352800EFEE76 /* PBXTextBookmark */; + 5F648F4617B8358B00EFEE76 = 5F648F4617B8358B00EFEE76 /* PBXTextBookmark */; + 5F648F4717B8358B00EFEE76 = 5F648F4717B8358B00EFEE76 /* PBXTextBookmark */; + 5F648F4E17B835DD00EFEE76 = 5F648F4E17B835DD00EFEE76 /* PBXTextBookmark */; + 5F648F5017B835DD00EFEE76 = 5F648F5017B835DD00EFEE76 /* PBXTextBookmark */; + 5F648F5117B835DD00EFEE76 = 5F648F5117B835DD00EFEE76 /* PBXTextBookmark */; + 5F648F5217B835DD00EFEE76 = 5F648F5217B835DD00EFEE76 /* PBXTextBookmark */; + 5F648F7417B839AD00EFEE76 = 5F648F7417B839AD00EFEE76 /* PBXBookmark */; + 5F648F7F17B83A6500EFEE76 = 5F648F7F17B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8017B83A6500EFEE76 = 5F648F8017B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8117B83A6500EFEE76 = 5F648F8117B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8217B83A6500EFEE76 = 5F648F8217B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8317B83A6500EFEE76 = 5F648F8317B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8417B83A6500EFEE76 = 5F648F8417B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8517B83A6500EFEE76 = 5F648F8517B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8617B83A6500EFEE76 = 5F648F8617B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8717B83A6500EFEE76 = 5F648F8717B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8817B83A6500EFEE76 = 5F648F8817B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8917B83A6500EFEE76 = 5F648F8917B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8A17B83A6500EFEE76 = 5F648F8A17B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8B17B83A6500EFEE76 = 5F648F8B17B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8E17B83A6500EFEE76 = 5F648F8E17B83A6500EFEE76 /* PBXTextBookmark */; + 5F648F8F17B83A6500EFEE76 = 5F648F8F17B83A6500EFEE76 /* PBXTextBookmark */; + 5F648FB517B83E0E00EFEE76 = 5F648FB517B83E0E00EFEE76 /* PBXTextBookmark */; + 5F648FB617B83E0E00EFEE76 = 5F648FB617B83E0E00EFEE76 /* PBXTextBookmark */; + 5F648FB717B83E0E00EFEE76 = 5F648FB717B83E0E00EFEE76 /* PBXTextBookmark */; + 5F648FB817B83E0E00EFEE76 = 5F648FB817B83E0E00EFEE76 /* PBXTextBookmark */; + 5F648FB917B83E0E00EFEE76 = 5F648FB917B83E0E00EFEE76 /* PBXTextBookmark */; + 5F648FBD17B83EF200EFEE76 = 5F648FBD17B83EF200EFEE76 /* PBXTextBookmark */; + 5F648FBE17B83EF200EFEE76 = 5F648FBE17B83EF200EFEE76 /* PBXTextBookmark */; + 5F648FBF17B83EF200EFEE76 = 5F648FBF17B83EF200EFEE76 /* PBXTextBookmark */; + 5F648FCD17B94D1C00EFEE76 = 5F648FCD17B94D1C00EFEE76 /* PBXBookmark */; + 5F648FD117B94E7100EFEE76 = 5F648FD117B94E7100EFEE76 /* PBXBookmark */; + 5F648FED17B9634C00EFEE76 = 5F648FED17B9634C00EFEE76 /* PBXBookmark */; + 5F648FFE17B9651E00EFEE76 = 5F648FFE17B9651E00EFEE76 /* PBXBookmark */; + 5F64900817B9691F00EFEE76 = 5F64900817B9691F00EFEE76 /* PBXBookmark */; + 5F64901317B969F000EFEE76 = 5F64901317B969F000EFEE76 /* PBXTextBookmark */; + 5F64901417B969F000EFEE76 = 5F64901417B969F000EFEE76 /* PBXTextBookmark */; + 5F64901517B969F000EFEE76 = 5F64901517B969F000EFEE76 /* PBXTextBookmark */; + 5F64901617B969F000EFEE76 = 5F64901617B969F000EFEE76 /* PBXTextBookmark */; + 5F64901717B969F000EFEE76 = 5F64901717B969F000EFEE76 /* PBXTextBookmark */; + 5F64901817B969F000EFEE76 = 5F64901817B969F000EFEE76 /* PBXTextBookmark */; + 5F64901917B969F000EFEE76 = 5F64901917B969F000EFEE76 /* PBXTextBookmark */; + 5F64901A17B969F000EFEE76 = 5F64901A17B969F000EFEE76 /* PBXTextBookmark */; + 5F64901B17B969F000EFEE76 = 5F64901B17B969F000EFEE76 /* PBXTextBookmark */; + 5F64901C17B969F000EFEE76 = 5F64901C17B969F000EFEE76 /* PBXTextBookmark */; + 5F64901D17B969F000EFEE76 = 5F64901D17B969F000EFEE76 /* PBXTextBookmark */; + 5F64901E17B969F000EFEE76 = 5F64901E17B969F000EFEE76 /* PBXTextBookmark */; + 5F64901F17B969F000EFEE76 = 5F64901F17B969F000EFEE76 /* PBXTextBookmark */; + 5F64902017B969F000EFEE76 = 5F64902017B969F000EFEE76 /* PBXTextBookmark */; + 5F64902317B969F000EFEE76 = 5F64902317B969F000EFEE76 /* PBXTextBookmark */; + 5F64902617B969F000EFEE76 = 5F64902617B969F000EFEE76 /* PBXTextBookmark */; + 5F64902717B969F000EFEE76 = 5F64902717B969F000EFEE76 /* PBXTextBookmark */; + 5F64902A17B969F000EFEE76 = 5F64902A17B969F000EFEE76 /* PBXTextBookmark */; + 5F64902D17B969F000EFEE76 = 5F64902D17B969F000EFEE76 /* PBXTextBookmark */; + 5F64902E17B969F000EFEE76 = 5F64902E17B969F000EFEE76 /* PBXTextBookmark */; + 5F64902F17B969F000EFEE76 = 5F64902F17B969F000EFEE76 /* PBXTextBookmark */; + 5F64903017B969F000EFEE76 = 5F64903017B969F000EFEE76 /* PBXTextBookmark */; + 5F64903317B969F000EFEE76 = 5F64903317B969F000EFEE76 /* PBXTextBookmark */; + 5F64903617B969F000EFEE76 = 5F64903617B969F000EFEE76 /* PBXTextBookmark */; + 5F64903917B969F000EFEE76 = 5F64903917B969F000EFEE76 /* PBXTextBookmark */; + 5F64903A17B969F000EFEE76 = 5F64903A17B969F000EFEE76 /* PBXTextBookmark */; + 5F64903B17B96A2700EFEE76 = 5F64903B17B96A2700EFEE76 /* PBXTextBookmark */; + 5F64903C17B96A2700EFEE76 = 5F64903C17B96A2700EFEE76 /* PBXTextBookmark */; + 5F64903D17B96A2700EFEE76 = 5F64903D17B96A2700EFEE76 /* PBXTextBookmark */; + 5F64903E17B96A2700EFEE76 = 5F64903E17B96A2700EFEE76 /* PBXTextBookmark */; + 5F64903F17B96A2700EFEE76 = 5F64903F17B96A2700EFEE76 /* PBXTextBookmark */; + 5F64904017B96A2700EFEE76 = 5F64904017B96A2700EFEE76 /* PBXTextBookmark */; + 5F64904117B96A2700EFEE76 = 5F64904117B96A2700EFEE76 /* PBXTextBookmark */; + 5F64904217B96A2700EFEE76 = 5F64904217B96A2700EFEE76 /* PBXTextBookmark */; + 5F64904317B96A2700EFEE76 = 5F64904317B96A2700EFEE76 /* PBXTextBookmark */; + 5F64904417B96A2700EFEE76 = 5F64904417B96A2700EFEE76 /* PBXTextBookmark */; + 5F64904D17B96AA300EFEE76 = 5F64904D17B96AA300EFEE76 /* PBXTextBookmark */; + 5F64904E17B96AA300EFEE76 = 5F64904E17B96AA300EFEE76 /* PBXTextBookmark */; + 5F64904F17B96AA300EFEE76 = 5F64904F17B96AA300EFEE76 /* PBXTextBookmark */; + 5F64905017B96AA300EFEE76 = 5F64905017B96AA300EFEE76 /* PBXTextBookmark */; + 5F64905117B96AA300EFEE76 = 5F64905117B96AA300EFEE76 /* PBXTextBookmark */; + 5F64905217B96AA300EFEE76 = 5F64905217B96AA300EFEE76 /* PBXTextBookmark */; + 5F64905317B96AA300EFEE76 = 5F64905317B96AA300EFEE76 /* PBXTextBookmark */; + 5F64905417B96AA300EFEE76 = 5F64905417B96AA300EFEE76 /* PBXTextBookmark */; + 5F64905517B96AA300EFEE76 = 5F64905517B96AA300EFEE76 /* PBXTextBookmark */; + 5F64905617B96AA300EFEE76 = 5F64905617B96AA300EFEE76 /* PBXTextBookmark */; + 5F64905717B96AA300EFEE76 = 5F64905717B96AA300EFEE76 /* PBXTextBookmark */; + 5F64905E17B96AEB00EFEE76 = 5F64905E17B96AEB00EFEE76 /* PBXTextBookmark */; + 5F64905F17B96AEB00EFEE76 = 5F64905F17B96AEB00EFEE76 /* PBXTextBookmark */; + 5F64906017B96AEB00EFEE76 = 5F64906017B96AEB00EFEE76 /* PBXTextBookmark */; + 5F64906117B96AEB00EFEE76 = 5F64906117B96AEB00EFEE76 /* PBXTextBookmark */; + 5F64906317B96AEB00EFEE76 = 5F64906317B96AEB00EFEE76 /* PBXTextBookmark */; + 5F64906517B96AEB00EFEE76 = 5F64906517B96AEB00EFEE76 /* PBXTextBookmark */; + 5F64906A17B96BB800EFEE76 = 5F64906A17B96BB800EFEE76 /* PBXTextBookmark */; + 5F64906B17B96BB800EFEE76 = 5F64906B17B96BB800EFEE76 /* PBXTextBookmark */; + 5F64906C17B96BB800EFEE76 = 5F64906C17B96BB800EFEE76 /* PBXTextBookmark */; + 5F64906D17B96BB800EFEE76 = 5F64906D17B96BB800EFEE76 /* PBXTextBookmark */; + 5F64906E17B96BB800EFEE76 = 5F64906E17B96BB800EFEE76 /* PBXTextBookmark */; + 5F64906F17B96BB800EFEE76 = 5F64906F17B96BB800EFEE76 /* PBXTextBookmark */; + 5F64907017B96BB800EFEE76 = 5F64907017B96BB800EFEE76 /* PBXTextBookmark */; + 5F64907117B96BB800EFEE76 = 5F64907117B96BB800EFEE76 /* PBXTextBookmark */; + 5F64907217B96BB800EFEE76 = 5F64907217B96BB800EFEE76 /* PBXTextBookmark */; + 5F64907317B96BB800EFEE76 = 5F64907317B96BB800EFEE76 /* PBXTextBookmark */; + 5F64907417B96BB800EFEE76 = 5F64907417B96BB800EFEE76 /* PBXTextBookmark */; + 5F64907517B96BB800EFEE76 = 5F64907517B96BB800EFEE76 /* PBXTextBookmark */; + 5F64907F17B96C1100EFEE76 = 5F64907F17B96C1100EFEE76 /* PBXTextBookmark */; + 5F64908017B96C1100EFEE76 = 5F64908017B96C1100EFEE76 /* PBXTextBookmark */; + 5F64908217B96C1100EFEE76 = 5F64908217B96C1100EFEE76 /* PBXTextBookmark */; + 5F64908517B96C1A00EFEE76 = 5F64908517B96C1A00EFEE76 /* PBXTextBookmark */; + 5F64908617B96C1A00EFEE76 = 5F64908617B96C1A00EFEE76 /* PBXTextBookmark */; + 5F64908717B96C1A00EFEE76 = 5F64908717B96C1A00EFEE76 /* PBXTextBookmark */; + 5F64908817B96C1A00EFEE76 = 5F64908817B96C1A00EFEE76 /* PBXTextBookmark */; + 5F64908917B96C1A00EFEE76 = 5F64908917B96C1A00EFEE76 /* PBXTextBookmark */; + 5F64908A17B96C1A00EFEE76 = 5F64908A17B96C1A00EFEE76 /* PBXTextBookmark */; + 5F64908B17B96C1A00EFEE76 = 5F64908B17B96C1A00EFEE76 /* PBXTextBookmark */; + 5F64908C17B96C1A00EFEE76 = 5F64908C17B96C1A00EFEE76 /* PBXTextBookmark */; + 5F64908D17B96C1A00EFEE76 = 5F64908D17B96C1A00EFEE76 /* PBXTextBookmark */; + 5F64908E17B96C1A00EFEE76 = 5F64908E17B96C1A00EFEE76 /* PBXTextBookmark */; + 5F64908F17B9743100EFEE76 = 5F64908F17B9743100EFEE76 /* PBXTextBookmark */; + 5F64909017B9743100EFEE76 = 5F64909017B9743100EFEE76 /* PBXTextBookmark */; + 5F64909117B9743100EFEE76 = 5F64909117B9743100EFEE76 /* PBXTextBookmark */; + 5F64909217B9743100EFEE76 = 5F64909217B9743100EFEE76 /* PBXTextBookmark */; + 5F64909317B9743100EFEE76 = 5F64909317B9743100EFEE76 /* PBXTextBookmark */; + 5F64909417B9743100EFEE76 = 5F64909417B9743100EFEE76 /* PBXTextBookmark */; + 5F64909517B9743100EFEE76 = 5F64909517B9743100EFEE76 /* PBXTextBookmark */; + 5F64909617B9743100EFEE76 = 5F64909617B9743100EFEE76 /* PBXTextBookmark */; + 5F64909717B9743100EFEE76 = 5F64909717B9743100EFEE76 /* PBXTextBookmark */; + 5F64909817B9743100EFEE76 = 5F64909817B9743100EFEE76 /* PBXTextBookmark */; + 5F64909917B9743100EFEE76 = 5F64909917B9743100EFEE76 /* PBXTextBookmark */; + 5F64909A17B9743100EFEE76 = 5F64909A17B9743100EFEE76 /* PBXTextBookmark */; + 5F64909B17B9743100EFEE76 = 5F64909B17B9743100EFEE76 /* PBXTextBookmark */; + 5F64909C17B9743100EFEE76 = 5F64909C17B9743100EFEE76 /* PBXTextBookmark */; + 5F6490A117B9748500EFEE76 = 5F6490A117B9748500EFEE76 /* PBXTextBookmark */; + 5F6490A217B9748500EFEE76 = 5F6490A217B9748500EFEE76 /* PBXTextBookmark */; + 5F6490A317B9748500EFEE76 = 5F6490A317B9748500EFEE76 /* PBXTextBookmark */; + 5F6490A417B9748500EFEE76 = 5F6490A417B9748500EFEE76 /* PBXTextBookmark */; + 5F6490A517B9748500EFEE76 = 5F6490A517B9748500EFEE76 /* PBXTextBookmark */; + 5F6490A617B9748500EFEE76 = 5F6490A617B9748500EFEE76 /* PBXTextBookmark */; + 5F6490A717B9748500EFEE76 = 5F6490A717B9748500EFEE76 /* PBXTextBookmark */; + 5F6490A817B9748500EFEE76 = 5F6490A817B9748500EFEE76 /* PBXTextBookmark */; + 5F6490A917B9748500EFEE76 = 5F6490A917B9748500EFEE76 /* PBXTextBookmark */; + 5F6490AA17B9748500EFEE76 = 5F6490AA17B9748500EFEE76 /* PBXTextBookmark */; + 5F6490AB17B9748500EFEE76 = 5F6490AB17B9748500EFEE76 /* PBXTextBookmark */; + 5F6490AC17B9748500EFEE76 = 5F6490AC17B9748500EFEE76 /* PBXTextBookmark */; + 5F6490AD17B974B200EFEE76 = 5F6490AD17B974B200EFEE76 /* PBXTextBookmark */; + 5F6490AE17B974B200EFEE76 = 5F6490AE17B974B200EFEE76 /* PBXTextBookmark */; + 5F6490AF17B974B200EFEE76 = 5F6490AF17B974B200EFEE76 /* PBXTextBookmark */; + 5F6490B017B974B200EFEE76 = 5F6490B017B974B200EFEE76 /* PBXTextBookmark */; + 5F6490B117B974B200EFEE76 = 5F6490B117B974B200EFEE76 /* PBXTextBookmark */; + 5F6490B217B974B200EFEE76 = 5F6490B217B974B200EFEE76 /* PBXTextBookmark */; + 5F6490B317B974B200EFEE76 = 5F6490B317B974B200EFEE76 /* PBXTextBookmark */; + 5F6490B417B974B200EFEE76 = 5F6490B417B974B200EFEE76 /* PBXTextBookmark */; + 5F6490B517B974B200EFEE76 = 5F6490B517B974B200EFEE76 /* PBXTextBookmark */; + 5F6490B617B974B200EFEE76 = 5F6490B617B974B200EFEE76 /* PBXTextBookmark */; + 5F6490B717B974B200EFEE76 = 5F6490B717B974B200EFEE76 /* PBXTextBookmark */; + 5F6490B817B974B200EFEE76 = 5F6490B817B974B200EFEE76 /* PBXTextBookmark */; + 5F6490B917B974C000EFEE76 = 5F6490B917B974C000EFEE76 /* PBXTextBookmark */; + 5F6490BA17B974C000EFEE76 = 5F6490BA17B974C000EFEE76 /* PBXTextBookmark */; + 5F6490BB17B974C000EFEE76 = 5F6490BB17B974C000EFEE76 /* PBXTextBookmark */; + 5F6490BC17B974C000EFEE76 = 5F6490BC17B974C000EFEE76 /* PBXTextBookmark */; + 5F6490BD17B974C000EFEE76 = 5F6490BD17B974C000EFEE76 /* PBXTextBookmark */; + 5F6490BE17B974C000EFEE76 = 5F6490BE17B974C000EFEE76 /* PBXTextBookmark */; + 5F6490BF17B974C000EFEE76 = 5F6490BF17B974C000EFEE76 /* PBXTextBookmark */; + 5F6490C017B974C000EFEE76 = 5F6490C017B974C000EFEE76 /* PBXTextBookmark */; + 5F6490C117B974C000EFEE76 = 5F6490C117B974C000EFEE76 /* PBXTextBookmark */; + 5F6490C217B974C000EFEE76 = 5F6490C217B974C000EFEE76 /* PBXTextBookmark */; + 5F6490C317B974C000EFEE76 = 5F6490C317B974C000EFEE76 /* PBXTextBookmark */; + 5F6490C417B974C000EFEE76 = 5F6490C417B974C000EFEE76 /* PBXTextBookmark */; + 5F6490C517BA91DE00EFEE76 = 5F6490C517BA91DE00EFEE76 /* PBXTextBookmark */; + 5F6490C617BA91DE00EFEE76 = 5F6490C617BA91DE00EFEE76 /* PBXTextBookmark */; + 5F6490C717BA91DE00EFEE76 = 5F6490C717BA91DE00EFEE76 /* PBXTextBookmark */; + 5F6490C817BA91DE00EFEE76 = 5F6490C817BA91DE00EFEE76 /* PBXTextBookmark */; + 5F6490C917BA91DE00EFEE76 = 5F6490C917BA91DE00EFEE76 /* PBXTextBookmark */; + 5F6490CA17BA91DE00EFEE76 = 5F6490CA17BA91DE00EFEE76 /* PBXTextBookmark */; + 5F6490CB17BA91DE00EFEE76 = 5F6490CB17BA91DE00EFEE76 /* PBXTextBookmark */; + 5F6490CC17BA91DE00EFEE76 = 5F6490CC17BA91DE00EFEE76 /* PBXTextBookmark */; + 5F6490CD17BA91DE00EFEE76 = 5F6490CD17BA91DE00EFEE76 /* PBXTextBookmark */; + 5F6490CE17BA91DE00EFEE76 = 5F6490CE17BA91DE00EFEE76 /* PBXTextBookmark */; + 5F6490CF17BA91DE00EFEE76 = 5F6490CF17BA91DE00EFEE76 /* PBXTextBookmark */; + 5F6490D017BA91DE00EFEE76 = 5F6490D017BA91DE00EFEE76 /* PBXTextBookmark */; + 5F6490D117BA966600EFEE76 = 5F6490D117BA966600EFEE76 /* PBXTextBookmark */; + 5F6490D217BA966600EFEE76 = 5F6490D217BA966600EFEE76 /* PBXTextBookmark */; + 5F6490D317BA966600EFEE76 = 5F6490D317BA966600EFEE76 /* PBXTextBookmark */; + 5F6490D417BA966600EFEE76 = 5F6490D417BA966600EFEE76 /* PBXTextBookmark */; + 5F6490D517BA966600EFEE76 = 5F6490D517BA966600EFEE76 /* PBXTextBookmark */; + 5F6490D617BA966600EFEE76 = 5F6490D617BA966600EFEE76 /* PBXTextBookmark */; + 5F6490D717BA966600EFEE76 = 5F6490D717BA966600EFEE76 /* PBXTextBookmark */; + 5F6490D817BA966600EFEE76 = 5F6490D817BA966600EFEE76 /* PBXTextBookmark */; + 5F6490D917BA966600EFEE76 = 5F6490D917BA966600EFEE76 /* PBXTextBookmark */; + 5F6490DA17BA966600EFEE76 = 5F6490DA17BA966600EFEE76 /* PBXTextBookmark */; + 5F6490DB17BA966600EFEE76 = 5F6490DB17BA966600EFEE76 /* PBXTextBookmark */; + 5F9451B417BBE4CF006070F8 /* PBXTextBookmark */ = 5F9451B417BBE4CF006070F8 /* PBXTextBookmark */; + 5F9451B517BBE4CF006070F8 /* PBXTextBookmark */ = 5F9451B517BBE4CF006070F8 /* PBXTextBookmark */; + 5F9451B617BBE4CF006070F8 /* XCBuildMessageTextBookmark */ = 5F9451B617BBE4CF006070F8 /* XCBuildMessageTextBookmark */; + 5F9451B717BBE4CF006070F8 /* PBXTextBookmark */ = 5F9451B717BBE4CF006070F8 /* PBXTextBookmark */; + 5F9451C117BBE51E006070F8 /* PBXTextBookmark */ = 5F9451C117BBE51E006070F8 /* PBXTextBookmark */; + 5F9451C217BBE51E006070F8 /* PBXTextBookmark */ = 5F9451C217BBE51E006070F8 /* PBXTextBookmark */; + 5F9451D517BBE5E9006070F8 /* PBXTextBookmark */ = 5F9451D517BBE5E9006070F8 /* PBXTextBookmark */; + 5F9451D617BBE5E9006070F8 /* PBXTextBookmark */ = 5F9451D617BBE5E9006070F8 /* PBXTextBookmark */; + 5F9451D717BBE5E9006070F8 /* PBXTextBookmark */ = 5F9451D717BBE5E9006070F8 /* PBXTextBookmark */; + 5F9451DC17BBE5F7006070F8 /* PBXTextBookmark */ = 5F9451DC17BBE5F7006070F8 /* PBXTextBookmark */; + 5F9451E117BBE65D006070F8 /* PBXTextBookmark */ = 5F9451E117BBE65D006070F8 /* PBXTextBookmark */; + 5F9451E217BBE65D006070F8 /* PBXTextBookmark */ = 5F9451E217BBE65D006070F8 /* PBXTextBookmark */; + 5F9451F017BBE8F2006070F8 /* PBXTextBookmark */ = 5F9451F017BBE8F2006070F8 /* PBXTextBookmark */; + 5F9451F117BBE8F2006070F8 /* PBXTextBookmark */ = 5F9451F117BBE8F2006070F8 /* PBXTextBookmark */; + 5F9451F317BBE91B006070F8 /* PBXTextBookmark */ = 5F9451F317BBE91B006070F8 /* PBXTextBookmark */; + 5F9451FC17BBEA66006070F8 /* PBXTextBookmark */ = 5F9451FC17BBEA66006070F8 /* PBXTextBookmark */; + 5F9451FD17BBEA66006070F8 /* PBXTextBookmark */ = 5F9451FD17BBEA66006070F8 /* PBXTextBookmark */; + 5F94520417BBEB7D006070F8 /* PBXTextBookmark */ = 5F94520417BBEB7D006070F8 /* PBXTextBookmark */; + 5F94520517BBEB7D006070F8 /* PBXTextBookmark */ = 5F94520517BBEB7D006070F8 /* PBXTextBookmark */; + 5F94520D17BBECF0006070F8 /* PBXTextBookmark */ = 5F94520D17BBECF0006070F8 /* PBXTextBookmark */; + 5F94520E17BBECF0006070F8 /* XCBuildMessageTextBookmark */ = 5F94520E17BBECF0006070F8 /* XCBuildMessageTextBookmark */; + 5F94521017BBECF0006070F8 /* PBXTextBookmark */ = 5F94521017BBECF0006070F8 /* PBXTextBookmark */; + 5F94521A17BBED7A006070F8 /* PBXTextBookmark */ = 5F94521A17BBED7A006070F8 /* PBXTextBookmark */; + 5F94521B17BBED82006070F8 /* PBXTextBookmark */ = 5F94521B17BBED82006070F8 /* PBXTextBookmark */; + 5F94521C17BBED82006070F8 /* PBXTextBookmark */ = 5F94521C17BBED82006070F8 /* PBXTextBookmark */; + 5F94521D17BBED82006070F8 /* PBXTextBookmark */ = 5F94521D17BBED82006070F8 /* PBXTextBookmark */; + 5F94521E17BBED82006070F8 /* PBXTextBookmark */ = 5F94521E17BBED82006070F8 /* PBXTextBookmark */; + 5F94522217BBEDC1006070F8 /* PBXTextBookmark */ = 5F94522217BBEDC1006070F8 /* PBXTextBookmark */; + 5F94522317BBEDC1006070F8 /* PBXTextBookmark */ = 5F94522317BBEDC1006070F8 /* PBXTextBookmark */; + 5F94522417BBEDC1006070F8 /* PBXTextBookmark */ = 5F94522417BBEDC1006070F8 /* PBXTextBookmark */; + 5F94522517BBEDC1006070F8 /* PBXTextBookmark */ = 5F94522517BBEDC1006070F8 /* PBXTextBookmark */; + 5F94522917BBEE1C006070F8 /* PBXBookmark */ = 5F94522917BBEE1C006070F8 /* PBXBookmark */; + 5F94522A17BBEE52006070F8 /* PBXTextBookmark */ = 5F94522A17BBEE52006070F8 /* PBXTextBookmark */; + 5F94522E17BBEEED006070F8 /* PBXTextBookmark */ = 5F94522E17BBEEED006070F8 /* PBXTextBookmark */; + 5F94523117BBEF2E006070F8 /* PBXTextBookmark */ = 5F94523117BBEF2E006070F8 /* PBXTextBookmark */; + 5F94523217BBEF2E006070F8 /* PBXTextBookmark */ = 5F94523217BBEF2E006070F8 /* PBXTextBookmark */; + 5F94523317BBEF2E006070F8 /* XCBuildMessageTextBookmark */ = 5F94523317BBEF2E006070F8 /* XCBuildMessageTextBookmark */; + 5F94523517BBEF2E006070F8 /* PBXTextBookmark */ = 5F94523517BBEF2E006070F8 /* PBXTextBookmark */; + 5F94523617BBEF32006070F8 /* PBXTextBookmark */ = 5F94523617BBEF32006070F8 /* PBXTextBookmark */; + 5F94523717BBEF32006070F8 /* PBXTextBookmark */ = 5F94523717BBEF32006070F8 /* PBXTextBookmark */; + 5F94523817BBEF32006070F8 /* PBXTextBookmark */ = 5F94523817BBEF32006070F8 /* PBXTextBookmark */; + 5F94523917BBEF32006070F8 /* PBXTextBookmark */ = 5F94523917BBEF32006070F8 /* PBXTextBookmark */; + 5F94523A17BBEF32006070F8 /* PBXTextBookmark */ = 5F94523A17BBEF32006070F8 /* PBXTextBookmark */; + 5F94523D17BBEF32006070F8 /* PBXTextBookmark */ = 5F94523D17BBEF32006070F8 /* PBXTextBookmark */; + 5F94524417BBEF66006070F8 /* PBXTextBookmark */ = 5F94524417BBEF66006070F8 /* PBXTextBookmark */; + 5F94524517BBEF68006070F8 /* PBXTextBookmark */ = 5F94524517BBEF68006070F8 /* PBXTextBookmark */; + 5F94524617BBEF68006070F8 /* PBXTextBookmark */ = 5F94524617BBEF68006070F8 /* PBXTextBookmark */; }; sourceControlManager = 5F7A0FE11589160900748D0D /* Source Control */; userBuildSettings = { @@ -332,9 +382,9 @@ }; 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {985, 2808}}"; - sepNavSelRange = "{6410, 46}"; - sepNavVisRange = "{4600, 2034}"; + sepNavIntBoundsRect = "{{0, 0}, {985, 3055}}"; + sepNavSelRange = "{957, 0}"; + sepNavVisRange = "{5163, 2013}"; sepNavWindowFrame = "{{38, 126}, {1022, 1026}}"; }; }; @@ -400,9 +450,9 @@ }; 5F3B0889158AC5520058A8B1 /* graphchi_engine.hpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1874, 15782}}"; - sepNavSelRange = "{47743, 0}"; - sepNavVisRange = "{27025, 2953}"; + sepNavIntBoundsRect = "{{0, 0}, {1874, 15379}}"; + sepNavSelRange = "{14447, 0}"; + sepNavVisRange = "{25727, 2803}"; sepNavWindowFrame = "{{84, 84}, {1022, 1026}}"; }; }; @@ -481,6 +531,12 @@ name = kwaymerge.hpp; path = ../src/util/kwaymerge.hpp; sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {961, 1326}}"; + sepNavSelRange = "{1198, 0}"; + sepNavVisRange = "{784, 1695}"; + sepNavWindowFrame = "{{15, 147}, {1022, 1026}}"; + }; }; 5F41F7AA17B006AE00C5FD90 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -523,6 +579,11 @@ name = ioutil.hpp; path = ../src/util/ioutil.hpp; sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {957, 3679}}"; + sepNavSelRange = "{5036, 0}"; + sepNavVisRange = "{4656, 913}"; + }; }; 5F41F80317B00BC600C5FD90 /* orderbydegree.hpp */ = { uiCtxt = { @@ -586,7 +647,7 @@ fRef = 5F41F7A917B006AE00C5FD90 /* kwaymerge.hpp */; name = "kwaymerge.hpp: 92"; rLen = 0; - rLoc = 2315; + rLoc = 2313; rType = 0; vrLen = 1569; vrLoc = 1050; @@ -717,16 +778,16 @@ fRef = 5F41F79E17B0068800C5FD90 /* kwaymerge.hpp */; name = "kwaymerge.hpp: 75"; rLen = 0; - rLoc = 1904; + rLoc = 1902; rType = 0; vrLen = 1455; vrLoc = 1143; }; 5F54B84715FD4C1900B3842C /* dynamicdata_smoketest.cpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {964, 2132}}"; - sepNavSelRange = "{4552, 0}"; - sepNavVisRange = "{3421, 1821}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 2093}}"; + sepNavSelRange = "{0, 0}"; + sepNavVisRange = "{0, 2324}"; sepNavWindowFrame = "{{38, 126}, {1022, 1026}}"; }; }; @@ -863,7 +924,7 @@ hitCount = 0; ignoreCount = 0; lineNumber = 236; - modificationTime = 398029895.166603; + modificationTime = 398192458.869713; originalNumberOfMultipleMatches = 0; state = 2; }; @@ -877,11 +938,10 @@ vrLen = 1346; vrLoc = 10031; }; - 5F648F0A17B8337200EFEE76 /* XCBuildMessageTextBookmark */ = { + 5F648F0A17B8337200EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; comments = "No matching function for call to 'remove_edgev(graphchi::graphchi_edge*)'"; fRef = 5F648F0B17B8337200EFEE76 /* graph_objects.hpp */; - fallbackIsa = XCBuildMessageTextBookmark; rLen = 1; rLoc = 354; rType = 1; @@ -892,11 +952,6 @@ name = graph_objects.hpp; path = ../src/api/graph_objects.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {961, 5382}}"; - sepNavSelRange = "{3007, 194}"; - sepNavVisRange = "{2114, 2128}"; - }; }; 5F648F0C17B8337200EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -928,11 +983,10 @@ vrLen = 2281; vrLoc = 8867; }; - 5F648F2717B8340900EFEE76 /* XCBuildMessageTextBookmark */ = { + 5F648F2717B8340900EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; comments = "There are no arguments to 'num_edges' that depend on a template parameter, so a declaration of 'num_edges' must be available"; fRef = 5F648F2817B8340900EFEE76 /* graph_objects.hpp */; - fallbackIsa = XCBuildMessageTextBookmark; rLen = 1; rLoc = 358; rType = 1; @@ -943,11 +997,6 @@ name = graph_objects.hpp; path = ../src/api/graph_objects.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {961, 5421}}"; - sepNavSelRange = "{3007, 194}"; - sepNavVisRange = "{2317, 2140}"; - }; }; 5F648F2917B8340900EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -1029,11 +1078,10 @@ vrLen = 2140; vrLoc = 2317; }; - 5F648F4E17B835DD00EFEE76 /* XCBuildMessageTextBookmark */ = { + 5F648F4E17B835DD00EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; comments = "There are no arguments to 'num_edges' that depend on a template parameter, so a declaration of 'num_edges' must be available"; fRef = 5F648F4F17B835DD00EFEE76 /* graph_objects.hpp */; - fallbackIsa = XCBuildMessageTextBookmark; rLen = 1; rLoc = 358; rType = 1; @@ -1044,11 +1092,6 @@ name = graph_objects.hpp; path = ../src/api/graph_objects.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1265, 5226}}"; - sepNavSelRange = "{7734, 215}"; - sepNavVisRange = "{7054, 2286}"; - }; }; 5F648F5017B835DD00EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -1161,7 +1204,7 @@ rLen = 0; rLoc = 4552; rType = 0; - vrLen = 1821; + vrLen = 1715; vrLoc = 3421; }; 5F648F8717B83A6500EFEE76 /* PBXTextBookmark */ = { @@ -1853,15 +1896,15 @@ }; 5F64906217B96AEB00EFEE76 /* stl_function.h */ = { isa = PBXFileReference; + lastKnownFileType = sourcecode.c.h; name = stl_function.h; path = "/usr/include/c++/4.2.1/bits/stl_function.h"; sourceTree = ""; }; - 5F64906317B96AEB00EFEE76 /* XCBuildMessageTextBookmark */ = { + 5F64906317B96AEB00EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; comments = "No match for 'operator<<' in 'std::operator<< [with _Traits = std::char_traits](((std::basic_ostream >&)((std::basic_ostream >*)std::cout. std::basic_ostream<_CharT, _Traits>::operator<< [with _CharT = char, _Traits = std::char_traits]((i + 1)))), ((const char*)\". label: \")) << curlabels. std::vector<_Tp, _Alloc>::operator[] [with _Tp = labelcount_tt, _Alloc = std::allocator >](((long unsigned int)i))->labelcount_tt::label'"; fRef = 5F64906417B96AEB00EFEE76 /* labelanalysis.hpp */; - fallbackIsa = XCBuildMessageTextBookmark; rLen = 1; rLoc = 181; rType = 1; @@ -1872,11 +1915,6 @@ name = labelanalysis.hpp; path = ../src/util/labelanalysis.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1268, 2483}}"; - sepNavSelRange = "{5616, 0}"; - sepNavVisRange = "{5076, 1326}"; - }; }; 5F64906517B96AEB00EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -2015,11 +2053,10 @@ vrLen = 1316; vrLoc = 5090; }; - 5F64908017B96C1100EFEE76 /* XCBuildMessageTextBookmark */ = { + 5F64908017B96C1100EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; comments = "No matching function for call to 'std::basic_ofstream >::open(const char*&, const char [2])'"; fRef = 5F64908117B96C1100EFEE76 /* labelanalysis.hpp */; - fallbackIsa = XCBuildMessageTextBookmark; rLen = 1; rLoc = 167; rType = 1; @@ -2030,11 +2067,6 @@ name = labelanalysis.hpp; path = ../src/util/labelanalysis.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {961, 2587}}"; - sepNavSelRange = "{0, 0}"; - sepNavVisRange = "{928, 2169}"; - }; }; 5F64908217B96C1100EFEE76 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -2876,6 +2908,13 @@ vrLen = 1704; vrLoc = 4362; }; + 5F67654116AF31C500359562 /* vertex_data_dynamic.hpp */ = { + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1268, 2977}}"; + sepNavSelRange = "{4437, 0}"; + sepNavVisRange = "{3458, 1713}"; + }; + }; 5F74B03A15D3532600ED3EA9 /* graphlab_lda */ = { activeExec = 0; executables = ( @@ -2892,9 +2931,9 @@ }; 5F74B04F15D6CEE200ED3EA9 /* blocksplitter.cpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {893, 1196}}"; + sepNavIntBoundsRect = "{{0, 0}, {964, 1196}}"; sepNavSelRange = "{2006, 0}"; - sepNavVisRange = "{714, 2291}"; + sepNavVisRange = "{498, 2507}"; }; }; 5F74B05215D6CF4F00ED3EA9 /* blocksplitter */ = { @@ -3063,9 +3102,9 @@ }; 5F7A10361589266800748D0D /* vertex_data.hpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {961, 1950}}"; - sepNavSelRange = "{2013, 76}"; - sepNavVisRange = "{1377, 2283}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 1950}}"; + sepNavSelRange = "{1918, 0}"; + sepNavVisRange = "{1337, 2412}"; }; }; 5F7A103B1589266800748D0D /* stripedio.hpp */ = { @@ -3077,9 +3116,9 @@ }; 5F7A103F1589266800748D0D /* metrics.hpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {915, 4407}}"; - sepNavSelRange = "{3864, 29}"; - sepNavVisRange = "{3710, 765}"; + sepNavIntBoundsRect = "{{0, 0}, {926, 4628}}"; + sepNavSelRange = "{6728, 0}"; + sepNavVisRange = "{6377, 631}"; }; }; 5F7A10471589266800748D0D /* memoryshard.hpp */ = { @@ -3184,6 +3223,539 @@ 5FCC20ED159DE4660003D0E9 /* connectedcomponents */, ); }; + 5F9451B217BBE4CF006070F8 /* conversions.hpp:264 */ = { + isa = PBXFileBreakpoint; + actions = ( + ); + breakpointStyle = 0; + continueAfterActions = 0; + countType = 0; + delayBeforeContinue = 0; + fileReference = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + functionName = "convert_edgelist(std::string inputfile, sharder &sharderobj, bool multivalue_edges=false)"; + hitCount = 0; + ignoreCount = 0; + lineNumber = 264; + modificationTime = 398192485.860185; + originalNumberOfMultipleMatches = 1; + state = 1; + }; + 5F9451B417BBE4CF006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F74B04F15D6CEE200ED3EA9 /* blocksplitter.cpp */; + name = "blocksplitter.cpp: 67"; + rLen = 0; + rLoc = 2006; + rType = 0; + vrLen = 2507; + vrLoc = 498; + }; + 5F9451B517BBE4CF006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 637"; + rLen = 0; + rLoc = 23727; + rType = 0; + vrLen = 4082; + vrLoc = 21897; + }; + 5F9451B617BBE4CF006070F8 /* XCBuildMessageTextBookmark */ = { + isa = PBXTextBookmark; + comments = "Expected unqualified-id before '<<' token"; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + fallbackIsa = XCBuildMessageTextBookmark; + rLen = 1; + rLoc = 656; + rType = 1; + }; + 5F9451B717BBE4CF006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 657"; + rLen = 0; + rLoc = 24118; + rType = 0; + vrLen = 3482; + vrLoc = 22625; + }; + 5F9451C117BBE51E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 657"; + rLen = 0; + rLoc = 24118; + rType = 0; + vrLen = 3482; + vrLoc = 22625; + }; + 5F9451C217BBE51E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 652"; + rLen = 0; + rLoc = 24636; + rType = 0; + vrLen = 4091; + vrLoc = 21897; + }; + 5F9451D517BBE5E9006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 720"; + rLen = 0; + rLoc = 27591; + rType = 0; + vrLen = 2033; + vrLoc = 26103; + }; + 5F9451D617BBE5E9006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 208"; + rLen = 46; + rLoc = 6410; + rType = 0; + vrLen = 1493; + vrLoc = 5153; + }; + 5F9451D717BBE5E9006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 33"; + rLen = 0; + rLoc = 956; + rType = 0; + vrLen = 1332; + vrLoc = 0; + }; + 5F9451DC17BBE5F7006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 33"; + rLen = 0; + rLoc = 956; + rType = 0; + vrLen = 1332; + vrLoc = 0; + }; + 5F9451E117BBE65D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 33"; + rLen = 0; + rLoc = 956; + rType = 0; + vrLen = 2292; + vrLoc = 4073; + }; + 5F9451E217BBE65D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 462"; + rLen = 0; + rLoc = 16136; + rType = 0; + vrLen = 2775; + vrLoc = 14252; + }; + 5F9451E417BBE83D006070F8 /* sharder.hpp:753 */ = { + isa = PBXFileBreakpoint; + actions = ( + ); + breakpointStyle = 0; + continueAfterActions = 0; + countType = 0; + delayBeforeContinue = 0; + fileReference = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + functionName = "createnextshard()"; + hitCount = 0; + ignoreCount = 0; + lineNumber = 753; + modificationTime = 398192485.860213; + originalNumberOfMultipleMatches = 1; + state = 1; + }; + 5F9451E917BBE8D2006070F8 /* sharder.hpp:451 */ = { + isa = PBXFileBreakpoint; + actions = ( + ); + breakpointStyle = 0; + continueAfterActions = 0; + countType = 0; + delayBeforeContinue = 0; + fileReference = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + functionName = "edata_flush(char * buf, char * bufptr, std::string & shard_filename, size_t totbytes)"; + hitCount = 0; + ignoreCount = 0; + lineNumber = 451; + modificationTime = 398192485.860199; + originalNumberOfMultipleMatches = 1; + state = 1; + }; + 5F9451F017BBE8F2006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 630"; + rLen = 0; + rLoc = 23001; + rType = 0; + vrLen = 2661; + vrLoc = 15893; + }; + 5F9451F117BBE8F2006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 465"; + rLen = 0; + rLoc = 16213; + rType = 0; + vrLen = 2667; + vrLoc = 15939; + }; + 5F9451F317BBE91B006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 465"; + rLen = 0; + rLoc = 16213; + rType = 0; + vrLen = 2739; + vrLoc = 15230; + }; + 5F9451FC17BBEA66006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 660"; + rLen = 0; + rLoc = 24328; + rType = 0; + vrLen = 1725; + vrLoc = 7194; + }; + 5F9451FD17BBEA66006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 660"; + rLen = 0; + rLoc = 24328; + rType = 0; + vrLen = 1725; + vrLoc = 7194; + }; + 5F94520417BBEB7D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 247"; + rLen = 0; + rLoc = 8411; + rType = 0; + vrLen = 2529; + vrLoc = 7192; + }; + 5F94520517BBEB7D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 750"; + rLen = 0; + rLoc = 28849; + rType = 0; + vrLen = 3810; + vrLoc = 27674; + }; + 5F94520D17BBECF0006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 660"; + rLen = 0; + rLoc = 24328; + rType = 0; + vrLen = 1604; + vrLoc = 7194; + }; + 5F94520E17BBECF0006070F8 /* XCBuildMessageTextBookmark */ = { + isa = PBXTextBookmark; + comments = "'intT' was not declared in this scope"; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + fallbackIsa = XCBuildMessageTextBookmark; + rLen = 1; + rLoc = 155; + rType = 1; + }; + 5F94520F17BBECF0006070F8 /* sharder.hpp */ = { + isa = PBXFileReference; + lastKnownFileType = sourcecode.cpp.h; + name = sharder.hpp; + path = ../src/preprocessing/dynamicdata/sharder.hpp; + sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1268, 11310}}"; + sepNavSelRange = "{5244, 0}"; + sepNavVisRange = "{4501, 2042}"; + }; + }; + 5F94521017BBECF0006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; + name = "sharder.hpp: 162"; + rLen = 0; + rLoc = 5244; + rType = 0; + vrLen = 2037; + vrLoc = 4679; + }; + 5F94521A17BBED7A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 156"; + rLen = 0; + rLoc = 5244; + rType = 0; + vrLen = 2042; + vrLoc = 4501; + }; + 5F94521B17BBED82006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 140"; + rLen = 0; + rLoc = 4495; + rType = 0; + vrLen = 3183; + vrLoc = 2768; + }; + 5F94521C17BBED82006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */; + name = "dynamicblock.hpp: 86"; + rLen = 0; + rLoc = 2719; + rType = 0; + vrLen = 2851; + vrLoc = 1647; + }; + 5F94521D17BBED82006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F7A10361589266800748D0D /* vertex_data.hpp */; + name = "vertex_data.hpp: 76"; + rLen = 76; + rLoc = 2013; + rType = 0; + vrLen = 2412; + vrLoc = 1337; + }; + 5F94521E17BBED82006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F7A10361589266800748D0D /* vertex_data.hpp */; + name = "vertex_data.hpp: 74"; + rLen = 0; + rLoc = 1918; + rType = 0; + vrLen = 2412; + vrLoc = 1337; + }; + 5F94522217BBEDC1006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F7A10361589266800748D0D /* vertex_data.hpp */; + name = "vertex_data.hpp: 74"; + rLen = 0; + rLoc = 1918; + rType = 0; + vrLen = 2412; + vrLoc = 1337; + }; + 5F94522317BBEDC1006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F67654116AF31C500359562 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 168"; + rLen = 0; + rLoc = 5412; + rType = 0; + vrLen = 2931; + vrLoc = 2997; + }; + 5F94522417BBEDC1006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */; + rLen = 0; + rLoc = 85; + rType = 1; + }; + 5F94522517BBEDC1006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */; + name = "dynamicblock.hpp: 86"; + rLen = 0; + rLoc = 2719; + rType = 0; + vrLen = 2978; + vrLoc = 1510; + }; + 5F94522917BBEE1C006070F8 /* PBXBookmark */ = { + isa = PBXBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + }; + 5F94522A17BBEE52006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 156"; + rLen = 0; + rLoc = 5244; + rType = 0; + vrLen = 2042; + vrLoc = 4501; + }; + 5F94522E17BBEEED006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 156"; + rLen = 0; + rLoc = 5244; + rType = 0; + vrLen = 2042; + vrLoc = 4501; + }; + 5F94523117BBEF2E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 156"; + rLen = 0; + rLoc = 5244; + rType = 0; + vrLen = 2042; + vrLoc = 4501; + }; + 5F94523217BBEF2E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F67654116AF31C500359562 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 144; + rLoc = 4295; + rType = 0; + vrLen = 1713; + vrLoc = 3458; + }; + 5F94523317BBEF2E006070F8 /* XCBuildMessageTextBookmark */ = { + isa = PBXTextBookmark; + comments = "Expected `)' before '(' token"; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + fallbackIsa = XCBuildMessageTextBookmark; + rLen = 1; + rLoc = 144; + rType = 1; + }; + 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */ = { + isa = PBXFileReference; + lastKnownFileType = sourcecode.cpp.h; + name = vertex_data_dynamic.hpp; + path = ../src/engine/auxdata/dynamicdata/vertex_data_dynamic.hpp; + sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1227, 3107}}"; + sepNavSelRange = "{4437, 0}"; + sepNavVisRange = "{2270, 2807}"; + }; + }; + 5F94523517BBEF2E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F67654116AF31C500359562 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 1713; + vrLoc = 3458; + }; + 5F94523617BBEF32006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F54B84715FD4C1900B3842C /* dynamicdata_smoketest.cpp */; + name = "dynamicdata_smoketest.cpp: 1"; + rLen = 0; + rLoc = 0; + rType = 0; + vrLen = 2324; + vrLoc = 0; + }; + 5F94523717BBEF32006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2120; + vrLoc = 15; + }; + 5F94523817BBEF32006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */; + name = "dynamicblock.hpp: 86"; + rLen = 0; + rLoc = 2719; + rType = 0; + vrLen = 2978; + vrLoc = 1510; + }; + 5F94523917BBEF32006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F67654116AF31C500359562 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 140"; + rLen = 685; + rLoc = 4106; + rType = 0; + vrLen = 2694; + vrLoc = 2465; + }; + 5F94523A17BBEF32006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 2737; + vrLoc = 2147; + }; + 5F94523D17BBEF32006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2013; + vrLoc = 5163; + }; + 5F94524417BBEF66006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 1804; + vrLoc = 3445; + }; + 5F94524517BBEF68006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 2807; + vrLoc = 2270; + }; + 5F94524617BBEF68006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2013; + vrLoc = 5163; + }; 5F9EBAAC1593F87D0092F7E2 /* chifilenames.hpp */ = { uiCtxt = { sepNavIntBoundsRect = "{{0, 0}, {894, 6487}}"; @@ -3218,8 +3790,23 @@ argumentStrings = ( ); autoAttachOnCrash = 1; - breakpointsEnabled = 0; + breakpointsEnabled = 1; configStateDict = { + "PBXLSLaunchAction-0" = { + PBXLSLaunchAction = 0; + PBXLSLaunchStartAction = 1; + PBXLSLaunchStdioStyle = 2; + PBXLSLaunchStyle = 0; + class = PBXLSRunLaunchConfig; + commandLineArgs = ( + ); + displayName = "Executable Runner"; + environment = { + }; + identifier = com.apple.Xcode.launch.runConfig; + remoteHostInfo = ""; + startActionInfo = ""; + }; }; customDataFormattersEnabled = 1; dataTipCustomDataFormattersEnabled = 1; @@ -3227,6 +3814,7 @@ dataTipSortType = 0; debuggerPlugin = GDBDebugging; disassemblyDisplayState = 0; + dylibVariantSuffix = ""; enableDebugStr = 1; environmentEntries = ( ); @@ -3234,9 +3822,14 @@ executableUserSymbolLevel = 0; libgmallocEnabled = 0; name = test_dynamicedata; + savedGlobals = { + }; showTypeColumn = 0; sourceDirectories = ( ); + startupPath = "/Users/akyrola/Projects/GraphCHI/graphchi-cpp/"; + variableFormatDictionary = { + }; }; 5FCB4691174AA11C0023D9D7 /* gensgd */ = { isa = PBXExecutable; @@ -3354,6 +3947,11 @@ name = conversions.hpp; path = ../src/preprocessing/conversions.hpp; sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1268, 10179}}"; + sepNavSelRange = "{24328, 0}"; + sepNavVisRange = "{7194, 1604}"; + }; }; 5FCC1DD71599194B0003D0E9 /* graphchi_basic_includes.hpp */ = { uiCtxt = { @@ -3643,11 +4241,6 @@ name = graph_objects.hpp; path = ../src/api/graph_objects.hpp; sourceTree = ""; - uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1268, 5213}}"; - sepNavSelRange = "{3809, 188}"; - sepNavVisRange = "{2761, 1480}"; - }; }; 5FCC229215A1EC330003D0E9 /* als_graphlab.cpp */ = { uiCtxt = { @@ -3808,9 +4401,23 @@ hitCount = 0; ignoreCount = 0; lineNumber = 267; - modificationTime = 398029895.166577; + modificationTime = 398192485.860164; originalNumberOfMultipleMatches = 1; - state = 0; + state = 1; + }; + 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */ = { + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1227, 1729}}"; + sepNavSelRange = "{2719, 0}"; + sepNavVisRange = "{1510, 2978}"; + }; + }; + 5FF7749917B3E1A200F84939 /* sharder.hpp */ = { + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1268, 11362}}"; + sepNavSelRange = "{5614, 0}"; + sepNavVisRange = "{4679, 2037}"; + }; }; 5FF774DD17B3E94100F84939 /* minimumspanningforest.cpp */ = { uiCtxt = { diff --git a/src/engine/auxdata/dynamicdata/vertex_data_dynamic.hpp b/src/engine/auxdata/dynamicdata/vertex_data_dynamic.hpp index 573f1b70..21d33050 100644 --- a/src/engine/auxdata/dynamicdata/vertex_data_dynamic.hpp +++ b/src/engine/auxdata/dynamicdata/vertex_data_dynamic.hpp @@ -142,9 +142,9 @@ namespace graphchi { std::string bfilename = blockfilename(blockid); if (!file_exists(bfilename)) { mkdir(dirname.c_str(), 0777); - size_t initsize = verticesperblock * sizeof(typename VertexDataType::sizeword_t); + size_t initsize = verticesperblock * (sizeof(typename VertexDataType::header_t) + sizeof(typename VertexDataType::sizeword_t)); int f = open(bfilename.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); - uint8_t * zeros = (uint8_t *) calloc(verticesperblock, sizeof(typename VertexDataType::sizeword_t)); + uint8_t * zeros = (uint8_t *) calloc(verticesperblock, sizeof(typename VertexDataType::header_t) + sizeof(typename VertexDataType::sizeword_t)); write_compressed(f, zeros, initsize); free(zeros); @@ -162,9 +162,10 @@ namespace graphchi { db.fd = iomgr->open_session(blockfname, false, true); int realsize = get_block_uncompressed_size(blockfname, -1); assert(realsize > 0); - iomgr->managed_malloc(db.fd, &db.data, realsize, 0); iomgr->managed_preada_now(db.fd, &db.data, realsize, 0); + assert(db.data); + db.dblock = new dynamicdata_block(verticesperblock, (uint8_t *)db.data, realsize); return db; } diff --git a/src/preprocessing/dynamicdata/sharder.hpp b/src/preprocessing/dynamicdata/sharder.hpp index 7ef169b4..f10e6d33 100644 --- a/src/preprocessing/dynamicdata/sharder.hpp +++ b/src/preprocessing/dynamicdata/sharder.hpp @@ -60,11 +60,8 @@ #include "shards/slidingshard.hpp" #include "output/output.hpp" #include "util/ioutil.hpp" -#include "util/radixSort.hpp" #include "util/kwaymerge.hpp" -#ifdef DYNAMICEDATA #include "util/qsort.hpp" -#endif namespace graphchi { @@ -83,7 +80,7 @@ namespace graphchi { // For dynamic edge data, we need to know if the value needs to be added // to the vector, or are we storing an empty vector. bool is_chivec_value; - edge_with_value() {} + edge_with_value() : value(std::vector(0)) {} edge_with_value(vid_t src, vid_t dst, std::vector value) : src(src), dst(dst), value(value) { } @@ -100,21 +97,27 @@ namespace graphchi { void reade(int f) { read(f, &src, sizeof(vid_t)); read(f, &dst, sizeof(vid_t)); - uint16_t nvalues; - read(f, &nvalues, sizeof(uint16_t)); - value.resize(nvalues); - read(f, &value[0], sizeof(VectorElementType) * nvalues); - read(f, &hdr, sizeof(HeaderDataType)); + read(f, &is_chivec_value, sizeof(bool)); + if (is_chivec_value) { + uint16_t nvalues; + read(f, &nvalues, sizeof(uint16_t)); + value.resize(nvalues); + read(f, &value[0], sizeof(VectorElementType) * nvalues); + read(f, &hdr, sizeof(HeaderDataType)); + } } void writee(int f) { writea(f, &src, sizeof(vid_t)); writea(f, &dst, sizeof(vid_t)); - uint16_t nvalues = value.size(); - assert(value.size() < 1<<16); - writea(f, &nvalues, sizeof(uint16_t)); - writea(f, &value[0], sizeof(VectorElementType) * nvalues); - writea(f, &hdr, sizeof(HeaderDataType)); + writea(f, &is_chivec_value, sizeof(bool)); + if (is_chivec_value) { + uint16_t nvalues = value.size(); + assert(value.size() < 1<<16); + writea(f, &nvalues, sizeof(uint16_t)); + writea(f, &value[0], sizeof(VectorElementType) * nvalues); + writea(f, &hdr, sizeof(HeaderDataType)); + } } @@ -134,13 +137,7 @@ namespace graphchi { return a.dst < b.dst; } - template - struct dstF {inline vid_t operator() (edge_with_value a) {return a.dst;} }; - - template - struct srcF {inline vid_t operator() (edge_with_value a) {return a.src;} }; - - + template struct shard_flushinfo { @@ -156,7 +153,7 @@ namespace graphchi { /* Sort */ // TODO: remove duplicates here! logstream(LOG_INFO) << "Sorting shovel: " << shovelname << ", max:" << max_vertex << std::endl; - iSort(buffer, (intT)numedges, (intT)max_vertex, dstF()); + quickSort(buffer, numedges, edge_t_dst_less); logstream(LOG_INFO) << "Sort done." << shovelname << std::endl; int f = open(shovelname.c_str(), O_WRONLY | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); //writea(f, buffer, numedges * sizeof(edge_with_value)); @@ -238,7 +235,7 @@ namespace graphchi { return idx < numedges; } - edge_with_value & next() { + edge_with_value next() { if (bufidx == bufsize_edges) { load_next(); } @@ -455,7 +452,7 @@ namespace graphchi { int f = open(block_filename.c_str(), O_RDWR | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR); write_compressed(f, buf, len); close(f); - + m.stop_time("edata_flush"); @@ -630,11 +627,11 @@ namespace graphchi { if (!edge.stopper()) { /* If we have dynamic edge data, we need to write the header of chivector - if there are edge values */ - bwrite_edata(ebuf, ebufptr, shovelbuf[i].hdr, tot_edatabytes, edfname, edgecounter); + bwrite_edata(ebuf, ebufptr, edge.hdr, tot_edatabytes, edfname, edgecounter); if (edge.is_chivec_value) { // Need to check how many values for this edge - int count = shovelbuf[i].value.size(); + int count = edge.value.size(); assert(count < 32768); typename chivector::sizeword_t szw; @@ -642,12 +639,12 @@ namespace graphchi { ((uint16_t *) &szw)[1] = (uint16_t)count; bwrite_edata::sizeword_t>(ebuf, ebufptr, szw, tot_edatabytes, edfname, edgecounter); for(int j=0; j < count; j++) { - bwrite_edata(ebuf, ebufptr, shovelbuf[i].value[j], tot_edatabytes, edfname, edgecounter); + bwrite_edata(ebuf, ebufptr, edge.value[j], tot_edatabytes, edfname, edgecounter); } - + } else { // Just write size word with zero - bwrite_edata(ebuf, ebufptr, 0, tot_edatabytes, edfname, edgecounter); + bwrite_edata::sizeword_t >(ebuf, ebufptr, 0, tot_edatabytes, edfname, edgecounter); } num_uniq_edges++; edgecounter++; // Increment edge counter here --- notice that dynamic edata case makes two or more calls to bwrite_edata before incrementing @@ -740,10 +737,12 @@ namespace graphchi { shard_capacity = (size_t) (1.2 * shard_capacity); sinkbuffer = (edge_with_value*) realloc(sinkbuffer, shard_capacity * sizeof(edge_with_value)); } - + sinkbuffer[cur_shard_counter++] = val; prevvid = val.dst; sharded_edges++; + + } void createnextshard() { @@ -754,6 +753,9 @@ namespace graphchi { sinkbuffer = (edge_with_value *) malloc(shard_capacity * sizeof(edge_with_value)); cur_shard_counter = 0; + std::cout << "Allocated sinkbuffer: " << shard_capacity * sizeof(edge_with_value) << ", shard_capacity=" << shard_capacity << std::endl; + std::cout << "sinkbuffer:" << sinkbuffer << std::endl; + // Adjust edges per hard so that it takes into account how many edges have been spilled now logstream(LOG_INFO) << "Remaining edges: " << (shoveled_edges - sharded_edges) << " remaining shards:" << (nshards - shardnum) << " edges per shard=" << edges_per_shard << std::endl; diff --git a/src/util/kwaymerge.hpp b/src/util/kwaymerge.hpp index 6dbea06d..a4924a60 100644 --- a/src/util/kwaymerge.hpp +++ b/src/util/kwaymerge.hpp @@ -41,7 +41,7 @@ template class merge_source { public: virtual bool has_more() = 0; - virtual T & next() = 0; + virtual T next() = 0; }; template class merge_sink { From efe43965a2dd1cbb82889039b0287432b2c636fa Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Wed, 14 Aug 2013 13:07:15 -0400 Subject: [PATCH 11/14] passed at least a small test --- .../graphchi_xcode.xcodeproj/akyrola.pbxuser | 201 ++++++++++++++++-- src/tests/test_dynamicedata_loader.cpp | 13 +- 2 files changed, 187 insertions(+), 27 deletions(-) diff --git a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser index 18107a65..9229e7c6 100644 --- a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser +++ b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser @@ -368,6 +368,20 @@ 5F94524417BBEF66006070F8 /* PBXTextBookmark */ = 5F94524417BBEF66006070F8 /* PBXTextBookmark */; 5F94524517BBEF68006070F8 /* PBXTextBookmark */ = 5F94524517BBEF68006070F8 /* PBXTextBookmark */; 5F94524617BBEF68006070F8 /* PBXTextBookmark */ = 5F94524617BBEF68006070F8 /* PBXTextBookmark */; + 5F94524D17BBF061006070F8 /* PBXTextBookmark */ = 5F94524D17BBF061006070F8 /* PBXTextBookmark */; + 5F94524E17BBF07A006070F8 /* PBXTextBookmark */ = 5F94524E17BBF07A006070F8 /* PBXTextBookmark */; + 5F94524F17BBF07A006070F8 /* PBXTextBookmark */ = 5F94524F17BBF07A006070F8 /* PBXTextBookmark */; + 5F94525017BBF07A006070F8 /* PBXTextBookmark */ = 5F94525017BBF07A006070F8 /* PBXTextBookmark */; + 5F94525117BBF07A006070F8 /* PBXTextBookmark */ = 5F94525117BBF07A006070F8 /* PBXTextBookmark */; + 5F94525217BBF07A006070F8 /* PBXTextBookmark */ = 5F94525217BBF07A006070F8 /* PBXTextBookmark */; + 5F94525317BBF07A006070F8 /* PBXTextBookmark */ = 5F94525317BBF07A006070F8 /* PBXTextBookmark */; + 5F94525417BBF07A006070F8 /* PBXTextBookmark */ = 5F94525417BBF07A006070F8 /* PBXTextBookmark */; + 5F94525617BBF0A8006070F8 /* PBXTextBookmark */ = 5F94525617BBF0A8006070F8 /* PBXTextBookmark */; + 5F94525717BBF0A8006070F8 /* PBXTextBookmark */ = 5F94525717BBF0A8006070F8 /* PBXTextBookmark */; + 5F94525817BBF0AC006070F8 /* PBXTextBookmark */ = 5F94525817BBF0AC006070F8 /* PBXTextBookmark */; + 5F94525917BBF0AC006070F8 /* PBXTextBookmark */ = 5F94525917BBF0AC006070F8 /* PBXTextBookmark */; + 5F94525C17BBF0BE006070F8 /* PBXTextBookmark */ = 5F94525C17BBF0BE006070F8 /* PBXTextBookmark */; + 5F94525D17BBF0BE006070F8 /* PBXTextBookmark */ = 5F94525D17BBF0BE006070F8 /* PBXTextBookmark */; }; sourceControlManager = 5F7A0FE11589160900748D0D /* Source Control */; userBuildSettings = { @@ -382,9 +396,9 @@ }; 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {985, 3055}}"; + sepNavIntBoundsRect = "{{0, 0}, {985, 3003}}"; sepNavSelRange = "{957, 0}"; - sepNavVisRange = "{5163, 2013}"; + sepNavVisRange = "{2636, 2396}"; sepNavWindowFrame = "{{38, 126}, {1022, 1026}}"; }; }; @@ -783,6 +797,13 @@ vrLen = 1455; vrLoc = 1143; }; + 5F54B84615FD4B2500B3842C /* chivector.hpp */ = { + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1227, 1820}}"; + sepNavSelRange = "{0, 0}"; + sepNavVisRange = "{1765, 1479}"; + }; + }; 5F54B84715FD4C1900B3842C /* dynamicdata_smoketest.cpp */ = { uiCtxt = { sepNavIntBoundsRect = "{{0, 0}, {1227, 2093}}"; @@ -924,7 +945,7 @@ hitCount = 0; ignoreCount = 0; lineNumber = 236; - modificationTime = 398192458.869713; + modificationTime = 398192817.828096; originalNumberOfMultipleMatches = 0; state = 2; }; @@ -3236,9 +3257,9 @@ hitCount = 0; ignoreCount = 0; lineNumber = 264; - modificationTime = 398192485.860185; + modificationTime = 398192817.828108; originalNumberOfMultipleMatches = 1; - state = 1; + state = 0; }; 5F9451B417BBE4CF006070F8 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -3314,7 +3335,7 @@ fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; name = "test_dynamicedata_loader.cpp: 208"; rLen = 46; - rLoc = 6410; + rLoc = 6492; rType = 0; vrLen = 1493; vrLoc = 5153; @@ -3372,9 +3393,9 @@ hitCount = 0; ignoreCount = 0; lineNumber = 753; - modificationTime = 398192485.860213; + modificationTime = 398192817.828127; originalNumberOfMultipleMatches = 1; - state = 1; + state = 0; }; 5F9451E917BBE8D2006070F8 /* sharder.hpp:451 */ = { isa = PBXFileBreakpoint; @@ -3389,9 +3410,9 @@ hitCount = 0; ignoreCount = 0; lineNumber = 451; - modificationTime = 398192485.860199; + modificationTime = 398192817.828136; originalNumberOfMultipleMatches = 1; - state = 1; + state = 0; }; 5F9451F017BBE8F2006070F8 /* PBXTextBookmark */ = { isa = PBXTextBookmark; @@ -3489,9 +3510,9 @@ path = ../src/preprocessing/dynamicdata/sharder.hpp; sourceTree = ""; uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1268, 11310}}"; - sepNavSelRange = "{5244, 0}"; - sepNavVisRange = "{4501, 2042}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 11635}}"; + sepNavSelRange = "{4495, 0}"; + sepNavVisRange = "{2768, 3183}"; }; }; 5F94521017BBECF0006070F8 /* PBXTextBookmark */ = { @@ -3651,9 +3672,9 @@ path = ../src/engine/auxdata/dynamicdata/vertex_data_dynamic.hpp; sourceTree = ""; uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1227, 3107}}"; + sepNavIntBoundsRect = "{{0, 0}, {1268, 2990}}"; sepNavSelRange = "{4437, 0}"; - sepNavVisRange = "{2270, 2807}"; + sepNavVisRange = "{3445, 1698}"; }; }; 5F94523517BBEF2E006070F8 /* PBXTextBookmark */ = { @@ -3756,6 +3777,146 @@ vrLen = 2013; vrLoc = 5163; }; + 5F94524D17BBF061006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 1698; + vrLoc = 3445; + }; + 5F94524E17BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 2786; + vrLoc = 2291; + }; + 5F94524F17BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 140"; + rLen = 0; + rLoc = 4495; + rType = 0; + vrLen = 3183; + vrLoc = 2768; + }; + 5F94525017BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F54B84615FD4B2500B3842C /* chivector.hpp */; + name = "chivector.hpp: 1"; + rLen = 0; + rLoc = 0; + rType = 0; + vrLen = 1479; + vrLoc = 1765; + }; + 5F94525117BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */; + name = "dynamicblock.hpp: 65"; + rLen = 0; + rLoc = 2165; + rType = 0; + vrLen = 2840; + vrLoc = 956; + }; + 5F94525217BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2129; + vrLoc = 6; + }; + 5F94525317BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2174; + vrLoc = 3575; + }; + 5F94525417BBF07A006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2346; + vrLoc = 2636; + }; + 5F94525617BBF0A8006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 158"; + rLen = 0; + rLoc = 5032; + rType = 0; + vrLen = 2170; + vrLoc = 3575; + }; + 5F94525717BBF0A8006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2346; + vrLoc = 2636; + }; + 5F94525817BBF0AC006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 158"; + rLen = 0; + rLoc = 5032; + rType = 0; + vrLen = 2174; + vrLoc = 3575; + }; + 5F94525917BBF0AC006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2394; + vrLoc = 2636; + }; + 5F94525C17BBF0BE006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 145"; + rLen = 0; + rLoc = 4503; + rType = 0; + vrLen = 2167; + vrLoc = 3586; + }; + 5F94525D17BBF0BE006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2396; + vrLoc = 2636; + }; 5F9EBAAC1593F87D0092F7E2 /* chifilenames.hpp */ = { uiCtxt = { sepNavIntBoundsRect = "{{0, 0}, {894, 6487}}"; @@ -4401,15 +4562,15 @@ hitCount = 0; ignoreCount = 0; lineNumber = 267; - modificationTime = 398192485.860164; + modificationTime = 398192817.828057; originalNumberOfMultipleMatches = 1; - state = 1; + state = 0; }; 5FE3D5AE15FD5E520061B039 /* dynamicblock.hpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1227, 1729}}"; - sepNavSelRange = "{2719, 0}"; - sepNavVisRange = "{1510, 2978}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 1833}}"; + sepNavSelRange = "{2165, 0}"; + sepNavVisRange = "{956, 2840}"; }; }; 5FF7749917B3E1A200F84939 /* sharder.hpp */ = { diff --git a/src/tests/test_dynamicedata_loader.cpp b/src/tests/test_dynamicedata_loader.cpp index 57405031..94b7dc98 100644 --- a/src/tests/test_dynamicedata_loader.cpp +++ b/src/tests/test_dynamicedata_loader.cpp @@ -84,10 +84,9 @@ struct DynamicDataLoaderTestProgram : public GraphChiProgramget(k) == expected); } - assert(evector->header().a == vertex.id() + vertex.edge(i)->vertex_id()); - assert(evector->header().b == gcontext.iteration %2); - evector->header().b = !evector->header().b; - + assert(evector->header().a >= vertex.id() + vertex.edge(i)->vertex_id() + gcontext.iteration); + assert(evector->header().b == (vertex.id() + vertex.edge(i)->vertex_id()) % 2); + evector->header().a = vertex.id() + vertex.edge(i)->vertex_id() + gcontext.iteration; lock.lock(); checksum += evector->get(0); lock.unlock(); @@ -143,7 +142,7 @@ int generatedata(std::string filename) { std::cout << "Generating data..." << std::endl; shouldbe = 0; - int totalVertices = 1000; + int totalVertices = 200000; for(int i=0; i < totalVertices; i++) { int nedges = random() % 10; for(int j=0; j < nedges; j++) { @@ -151,15 +150,15 @@ int generatedata(std::string filename) { if (dst != i) { std::vector edgevec; if ((i + dst) % 3 == 1) { + edgevec.push_back(i + dst); edgevec.push_back(i + dst + 1); edgevec.push_back(i + dst + 2); } else { edgevec.push_back(i + dst); - edgevec.push_back(i + dst + 1); } shouldbe += 2 * (i + dst); - sharderobj.preprocessing_add_edge_multival(i, dst, header_t(i + dst, true), edgevec); + sharderobj.preprocessing_add_edge_multival(i, dst, header_t(i + dst, (i + dst) % 2), edgevec); } } } From 8c6564abccc5ee04bbe45e04dae7012eac311d8a Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Wed, 14 Aug 2013 13:28:19 -0400 Subject: [PATCH 12/14] compiling version --- example_apps/randomwalks.cpp | 2 +- .../graphchi_xcode.xcodeproj/akyrola.pbxuser | 388 ++++++++++++++---- src/api/dynamicdata/chivector.hpp | 3 +- src/preprocessing/conversions.hpp | 1 + src/preprocessing/dynamicdata/sharder.hpp | 29 +- src/preprocessing/sharder.hpp | 2 +- 6 files changed, 335 insertions(+), 90 deletions(-) diff --git a/example_apps/randomwalks.cpp b/example_apps/randomwalks.cpp index 39506809..25104890 100644 --- a/example_apps/randomwalks.cpp +++ b/example_apps/randomwalks.cpp @@ -154,7 +154,7 @@ int main(int argc, const char ** argv) { /* Detect the number of shards or preprocess an input to create them */ bool preexisting_shards; - int nshards = convert_if_notexists(filename, get_option_string("nshards", "auto"), preexisting_shards); + int nshards = convert_if_notexists(filename, get_option_string("nshards", "auto"), preexisting_shards); /* Run */ RandomWalkProgram program; diff --git a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser index 9229e7c6..8f887fb9 100644 --- a/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser +++ b/graphchi_xcode/graphchi_xcode.xcodeproj/akyrola.pbxuser @@ -10,9 +10,8 @@ breakpoints = ( 5FCC250915A392C70003D0E9 /* streaming_pagerank.cpp:267 */, 5F648F0617B8332100EFEE76 /* stronglyconnectedcomponents.cpp:236 */, - 5F9451B217BBE4CF006070F8 /* conversions.hpp:264 */, - 5F9451E417BBE83D006070F8 /* sharder.hpp:753 */, - 5F9451E917BBE8D2006070F8 /* sharder.hpp:451 */, + 5F9451E417BBE83D006070F8 /* sharder.hpp:756 */, + 5F9451E917BBE8D2006070F8 /* sharder.hpp:454 */, ); codeSenseManager = 5F7A0FE21589160900748D0D /* Code sense */; executables = ( @@ -382,6 +381,29 @@ 5F94525917BBF0AC006070F8 /* PBXTextBookmark */ = 5F94525917BBF0AC006070F8 /* PBXTextBookmark */; 5F94525C17BBF0BE006070F8 /* PBXTextBookmark */ = 5F94525C17BBF0BE006070F8 /* PBXTextBookmark */; 5F94525D17BBF0BE006070F8 /* PBXTextBookmark */ = 5F94525D17BBF0BE006070F8 /* PBXTextBookmark */; + 5F94526017BBF30C006070F8 /* PBXTextBookmark */ = 5F94526017BBF30C006070F8 /* PBXTextBookmark */; + 5F94526117BBF30C006070F8 /* PBXTextBookmark */ = 5F94526117BBF30C006070F8 /* PBXTextBookmark */; + 5F94526217BBF30C006070F8 /* PBXTextBookmark */ = 5F94526217BBF30C006070F8 /* PBXTextBookmark */; + 5F94526317BBF30C006070F8 /* PBXTextBookmark */ = 5F94526317BBF30C006070F8 /* PBXTextBookmark */; + 5F94526417BBF359006070F8 /* PBXTextBookmark */ = 5F94526417BBF359006070F8 /* PBXTextBookmark */; + 5F94526517BBF359006070F8 /* PBXTextBookmark */ = 5F94526517BBF359006070F8 /* PBXTextBookmark */; + 5F94526617BBF359006070F8 /* PBXTextBookmark */ = 5F94526617BBF359006070F8 /* PBXTextBookmark */; + 5F94526717BBF359006070F8 /* PBXTextBookmark */ = 5F94526717BBF359006070F8 /* PBXTextBookmark */; + 5F94526817BBF38D006070F8 /* PBXTextBookmark */ = 5F94526817BBF38D006070F8 /* PBXTextBookmark */; + 5F94526917BBF38D006070F8 /* PBXTextBookmark */ = 5F94526917BBF38D006070F8 /* PBXTextBookmark */; + 5F94526A17BBF38D006070F8 /* PBXTextBookmark */ = 5F94526A17BBF38D006070F8 /* PBXTextBookmark */; + 5F94526B17BBF38D006070F8 /* PBXTextBookmark */ = 5F94526B17BBF38D006070F8 /* PBXTextBookmark */; + 5F94526C17BBF45F006070F8 /* PBXTextBookmark */ = 5F94526C17BBF45F006070F8 /* PBXTextBookmark */; + 5F94526D17BBF45F006070F8 /* PBXTextBookmark */ = 5F94526D17BBF45F006070F8 /* PBXTextBookmark */; + 5F94526E17BBF45F006070F8 /* PBXTextBookmark */ = 5F94526E17BBF45F006070F8 /* PBXTextBookmark */; + 5F94526F17BBF45F006070F8 /* PBXTextBookmark */ = 5F94526F17BBF45F006070F8 /* PBXTextBookmark */; + 5F94527017BBF45F006070F8 /* PBXTextBookmark */ = 5F94527017BBF45F006070F8 /* PBXTextBookmark */; + 5F94527717BBF508006070F8 /* PBXTextBookmark */ = 5F94527717BBF508006070F8 /* PBXTextBookmark */; + 5F94527817BBF508006070F8 /* PBXTextBookmark */ = 5F94527817BBF508006070F8 /* PBXTextBookmark */; + 5F94527A17BBF52E006070F8 /* PBXTextBookmark */ = 5F94527A17BBF52E006070F8 /* PBXTextBookmark */; + 5F94527B17BBF52E006070F8 /* PBXTextBookmark */ = 5F94527B17BBF52E006070F8 /* PBXTextBookmark */; + 5F94527C17BBF52E006070F8 /* PBXTextBookmark */ = 5F94527C17BBF52E006070F8 /* PBXTextBookmark */; + 5F94527D17BBF52E006070F8 /* PBXTextBookmark */ = 5F94527D17BBF52E006070F8 /* PBXTextBookmark */; }; sourceControlManager = 5F7A0FE11589160900748D0D /* Source Control */; userBuildSettings = { @@ -389,16 +411,16 @@ }; 5F0A1FBC16A9FBA50066FB56 /* randomwalks.cpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {964, 2431}}"; - sepNavSelRange = "{0, 0}"; - sepNavVisRange = "{0, 2191}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 2327}}"; + sepNavSelRange = "{5544, 0}"; + sepNavVisRange = "{3852, 2439}"; }; }; 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */ = { uiCtxt = { sepNavIntBoundsRect = "{{0, 0}, {985, 3003}}"; sepNavSelRange = "{957, 0}"; - sepNavVisRange = "{2636, 2396}"; + sepNavVisRange = "{2636, 2348}"; sepNavWindowFrame = "{{38, 126}, {1022, 1026}}"; }; }; @@ -485,7 +507,7 @@ fRef = 5F78366C15DD718A00954509 /* sharder.hpp */; name = "sharder.hpp: 658"; rLen = 0; - rLoc = 23495; + rLoc = 23493; rType = 0; vrLen = 3755; vrLoc = 21833; @@ -565,7 +587,7 @@ fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; name = "conversions.hpp: 594"; rLen = 0; - rLoc = 22020; + rLoc = 22021; rType = 0; vrLen = 3496; vrLoc = 20430; @@ -762,7 +784,7 @@ fRef = 5FCC1D091595000F0003D0E9 /* conversions.hpp */; name = "conversions.hpp: 502"; rLen = 0; - rLoc = 18207; + rLoc = 18208; rType = 0; vrLen = 2370; vrLoc = 17609; @@ -799,9 +821,9 @@ }; 5F54B84615FD4B2500B3842C /* chivector.hpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1227, 1820}}"; - sepNavSelRange = "{0, 0}"; - sepNavVisRange = "{1765, 1479}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 1742}}"; + sepNavSelRange = "{1067, 0}"; + sepNavVisRange = "{0, 1982}"; }; }; 5F54B84715FD4C1900B3842C /* dynamicdata_smoketest.cpp */ = { @@ -945,7 +967,7 @@ hitCount = 0; ignoreCount = 0; lineNumber = 236; - modificationTime = 398192817.828096; + modificationTime = 398193925.409738; originalNumberOfMultipleMatches = 0; state = 2; }; @@ -3023,6 +3045,11 @@ name = sharder.hpp; path = ../src/preprocessing/sharder.hpp; sourceTree = ""; + uiCtxt = { + sepNavIntBoundsRect = "{{0, 0}, {1227, 13962}}"; + sepNavSelRange = "{6538, 0}"; + sepNavVisRange = "{4621, 2481}"; + }; }; 5F7A0FE11589160900748D0D /* Source Control */ = { isa = PBXSourceControlManager; @@ -3244,23 +3271,6 @@ 5FCC20ED159DE4660003D0E9 /* connectedcomponents */, ); }; - 5F9451B217BBE4CF006070F8 /* conversions.hpp:264 */ = { - isa = PBXFileBreakpoint; - actions = ( - ); - breakpointStyle = 0; - continueAfterActions = 0; - countType = 0; - delayBeforeContinue = 0; - fileReference = 5FCC1D65159503890003D0E9 /* conversions.hpp */; - functionName = "convert_edgelist(std::string inputfile, sharder &sharderobj, bool multivalue_edges=false)"; - hitCount = 0; - ignoreCount = 0; - lineNumber = 264; - modificationTime = 398192817.828108; - originalNumberOfMultipleMatches = 1; - state = 0; - }; 5F9451B417BBE4CF006070F8 /* PBXTextBookmark */ = { isa = PBXTextBookmark; fRef = 5F74B04F15D6CEE200ED3EA9 /* blocksplitter.cpp */; @@ -3276,7 +3286,7 @@ fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; name = "sharder.hpp: 637"; rLen = 0; - rLoc = 23727; + rLoc = 24279; rType = 0; vrLen = 4082; vrLoc = 21897; @@ -3287,7 +3297,7 @@ fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; fallbackIsa = XCBuildMessageTextBookmark; rLen = 1; - rLoc = 656; + rLoc = 657; rType = 1; }; 5F9451B717BBE4CF006070F8 /* PBXTextBookmark */ = { @@ -3295,7 +3305,7 @@ fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; name = "conversions.hpp: 657"; rLen = 0; - rLoc = 24118; + rLoc = 24119; rType = 0; vrLen = 3482; vrLoc = 22625; @@ -3305,7 +3315,7 @@ fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; name = "conversions.hpp: 657"; rLen = 0; - rLoc = 24118; + rLoc = 24119; rType = 0; vrLen = 3482; vrLoc = 22625; @@ -3315,7 +3325,7 @@ fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; name = "sharder.hpp: 652"; rLen = 0; - rLoc = 24636; + rLoc = 25188; rType = 0; vrLen = 4091; vrLoc = 21897; @@ -3325,7 +3335,7 @@ fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; name = "sharder.hpp: 720"; rLen = 0; - rLoc = 27591; + rLoc = 28143; rType = 0; vrLen = 2033; vrLoc = 26103; @@ -3375,12 +3385,12 @@ fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; name = "sharder.hpp: 462"; rLen = 0; - rLoc = 16136; + rLoc = 16688; rType = 0; vrLen = 2775; vrLoc = 14252; }; - 5F9451E417BBE83D006070F8 /* sharder.hpp:753 */ = { + 5F9451E417BBE83D006070F8 /* sharder.hpp:756 */ = { isa = PBXFileBreakpoint; actions = ( ); @@ -3392,12 +3402,12 @@ functionName = "createnextshard()"; hitCount = 0; ignoreCount = 0; - lineNumber = 753; - modificationTime = 398192817.828127; + lineNumber = 756; + modificationTime = 398193925.409763; originalNumberOfMultipleMatches = 1; state = 0; }; - 5F9451E917BBE8D2006070F8 /* sharder.hpp:451 */ = { + 5F9451E917BBE8D2006070F8 /* sharder.hpp:454 */ = { isa = PBXFileBreakpoint; actions = ( ); @@ -3409,8 +3419,8 @@ functionName = "edata_flush(char * buf, char * bufptr, std::string & shard_filename, size_t totbytes)"; hitCount = 0; ignoreCount = 0; - lineNumber = 451; - modificationTime = 398192817.828136; + lineNumber = 454; + modificationTime = 398193925.409929; originalNumberOfMultipleMatches = 1; state = 0; }; @@ -3419,7 +3429,7 @@ fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; name = "sharder.hpp: 630"; rLen = 0; - rLoc = 23001; + rLoc = 23553; rType = 0; vrLen = 2661; vrLoc = 15893; @@ -3429,7 +3439,7 @@ fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; name = "sharder.hpp: 465"; rLen = 0; - rLoc = 16213; + rLoc = 16765; rType = 0; vrLen = 2667; vrLoc = 15939; @@ -3439,7 +3449,7 @@ fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; name = "sharder.hpp: 465"; rLen = 0; - rLoc = 16213; + rLoc = 16765; rType = 0; vrLen = 2739; vrLoc = 15230; @@ -3449,7 +3459,7 @@ fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; name = "conversions.hpp: 660"; rLen = 0; - rLoc = 24328; + rLoc = 24329; rType = 0; vrLen = 1725; vrLoc = 7194; @@ -3459,7 +3469,7 @@ fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; name = "conversions.hpp: 660"; rLen = 0; - rLoc = 24328; + rLoc = 24329; rType = 0; vrLen = 1725; vrLoc = 7194; @@ -3469,7 +3479,7 @@ fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; name = "sharder.hpp: 247"; rLen = 0; - rLoc = 8411; + rLoc = 8963; rType = 0; vrLen = 2529; vrLoc = 7192; @@ -3479,7 +3489,7 @@ fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; name = "sharder.hpp: 750"; rLen = 0; - rLoc = 28849; + rLoc = 29401; rType = 0; vrLen = 3810; vrLoc = 27674; @@ -3489,7 +3499,7 @@ fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; name = "conversions.hpp: 660"; rLen = 0; - rLoc = 24328; + rLoc = 24329; rType = 0; vrLen = 1604; vrLoc = 7194; @@ -3500,7 +3510,7 @@ fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; fallbackIsa = XCBuildMessageTextBookmark; rLen = 1; - rLoc = 155; + rLoc = 158; rType = 1; }; 5F94520F17BBECF0006070F8 /* sharder.hpp */ = { @@ -3510,9 +3520,9 @@ path = ../src/preprocessing/dynamicdata/sharder.hpp; sourceTree = ""; uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1227, 11635}}"; - sepNavSelRange = "{4495, 0}"; - sepNavVisRange = "{2768, 3183}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 11687}}"; + sepNavSelRange = "{4348, 0}"; + sepNavVisRange = "{1230, 3143}"; }; }; 5F94521017BBECF0006070F8 /* PBXTextBookmark */ = { @@ -3520,7 +3530,7 @@ fRef = 5FF7749917B3E1A200F84939 /* sharder.hpp */; name = "sharder.hpp: 162"; rLen = 0; - rLoc = 5244; + rLoc = 5796; rType = 0; vrLen = 2037; vrLoc = 4679; @@ -3530,7 +3540,7 @@ fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; name = "sharder.hpp: 156"; rLen = 0; - rLoc = 5244; + rLoc = 5796; rType = 0; vrLen = 2042; vrLoc = 4501; @@ -3540,7 +3550,7 @@ fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; name = "sharder.hpp: 140"; rLen = 0; - rLoc = 4495; + rLoc = 5047; rType = 0; vrLen = 3183; vrLoc = 2768; @@ -3621,7 +3631,7 @@ fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; name = "sharder.hpp: 156"; rLen = 0; - rLoc = 5244; + rLoc = 5796; rType = 0; vrLen = 2042; vrLoc = 4501; @@ -3631,7 +3641,7 @@ fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; name = "sharder.hpp: 156"; rLen = 0; - rLoc = 5244; + rLoc = 5796; rType = 0; vrLen = 2042; vrLoc = 4501; @@ -3641,7 +3651,7 @@ fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; name = "sharder.hpp: 156"; rLen = 0; - rLoc = 5244; + rLoc = 5796; rType = 0; vrLen = 2042; vrLoc = 4501; @@ -3672,9 +3682,9 @@ path = ../src/engine/auxdata/dynamicdata/vertex_data_dynamic.hpp; sourceTree = ""; uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1268, 2990}}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 3042}}"; sepNavSelRange = "{4437, 0}"; - sepNavVisRange = "{3445, 1698}"; + sepNavVisRange = "{2279, 2864}"; }; }; 5F94523517BBEF2E006070F8 /* PBXTextBookmark */ = { @@ -3802,7 +3812,7 @@ fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; name = "sharder.hpp: 140"; rLen = 0; - rLoc = 4495; + rLoc = 5047; rType = 0; vrLen = 3183; vrLoc = 2768; @@ -3917,6 +3927,236 @@ vrLen = 2396; vrLoc = 2636; }; + 5F94526017BBF30C006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 145"; + rLen = 0; + rLoc = 4503; + rType = 0; + vrLen = 2600; + vrLoc = 426; + }; + 5F94526117BBF30C006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94523417BBEF2E006070F8 /* vertex_data_dynamic.hpp */; + name = "vertex_data_dynamic.hpp: 145"; + rLen = 0; + rLoc = 4437; + rType = 0; + vrLen = 2864; + vrLoc = 2279; + }; + 5F94526217BBF30C006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F54B84615FD4B2500B3842C /* chivector.hpp */; + name = "chivector.hpp: 36"; + rLen = 0; + rLoc = 1067; + rType = 0; + vrLen = 1982; + vrLoc = 0; + }; + 5F94526317BBF30C006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2396; + vrLoc = 2636; + }; + 5F94526417BBF359006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F54B84615FD4B2500B3842C /* chivector.hpp */; + name = "chivector.hpp: 36"; + rLen = 0; + rLoc = 1067; + rType = 0; + vrLen = 1982; + vrLoc = 0; + }; + 5F94526517BBF359006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F78366C15DD718A00954509 /* sharder.hpp */; + name = "sharder.hpp: 1"; + rLen = 0; + rLoc = 0; + rType = 0; + vrLen = 1797; + vrLoc = 750; + }; + 5F94526617BBF359006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 88"; + rLen = 0; + rLoc = 2570; + rType = 0; + vrLen = 2684; + vrLoc = 1211; + }; + 5F94526717BBF359006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2396; + vrLoc = 2636; + }; + 5F94526817BBF38D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 88"; + rLen = 0; + rLoc = 2570; + rType = 0; + vrLen = 2684; + vrLoc = 1211; + }; + 5F94526917BBF38D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 657"; + rLen = 0; + rLoc = 24119; + rType = 0; + vrLen = 3315; + vrLoc = 22625; + }; + 5F94526A17BBF38D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D091595000F0003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 657"; + rLen = 0; + rLoc = 24119; + rType = 0; + vrLen = 2739; + vrLoc = 6375; + }; + 5F94526B17BBF38D006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2396; + vrLoc = 2636; + }; + 5F94526C17BBF45F006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D65159503890003D0E9 /* conversions.hpp */; + name = "conversions.hpp: 281"; + rLen = 0; + rLoc = 8950; + rType = 0; + vrLen = 3001; + vrLoc = 5981; + }; + 5F94526D17BBF45F006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FBC16A9FBA50066FB56 /* randomwalks.cpp */; + name = "randomwalks.cpp: 157"; + rLen = 0; + rLoc = 5544; + rType = 0; + vrLen = 2439; + vrLoc = 3852; + }; + 5F94526E17BBF45F006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 88"; + rLen = 0; + rLoc = 2570; + rType = 0; + vrLen = 2666; + vrLoc = 1229; + }; + 5F94526F17BBF45F006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 88"; + rLen = 0; + rLoc = 2569; + rType = 0; + vrLen = 2665; + vrLoc = 1229; + }; + 5F94527017BBF45F006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2396; + vrLoc = 2636; + }; + 5F94527717BBF508006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 122"; + rLen = 0; + rLoc = 4348; + rType = 0; + vrLen = 3143; + vrLoc = 1230; + }; + 5F94527817BBF508006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2348; + vrLoc = 2636; + }; + 5F94527A17BBF52E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F94520F17BBECF0006070F8 /* sharder.hpp */; + name = "sharder.hpp: 122"; + rLen = 0; + rLoc = 4348; + rType = 0; + vrLen = 3143; + vrLoc = 1230; + }; + 5F94527B17BBF52E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5FCC1D041594FADE0003D0E9 /* sharder_basic.cpp */; + name = "sharder_basic.cpp: 64"; + rLen = 0; + rLoc = 2323; + rType = 0; + vrLen = 2961; + vrLoc = 13; + }; + 5F94527C17BBF52E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F78366C15DD718A00954509 /* sharder.hpp */; + name = "sharder.hpp: 207"; + rLen = 0; + rLoc = 6538; + rType = 0; + vrLen = 2481; + vrLoc = 4621; + }; + 5F94527D17BBF52E006070F8 /* PBXTextBookmark */ = { + isa = PBXTextBookmark; + fRef = 5F0A1FCD16AB3E4E0066FB56 /* test_dynamicedata_loader.cpp */; + name = "test_dynamicedata_loader.cpp: 34"; + rLen = 0; + rLoc = 957; + rType = 0; + vrLen = 2348; + vrLoc = 2636; + }; 5F9EBAAC1593F87D0092F7E2 /* chifilenames.hpp */ = { uiCtxt = { sepNavIntBoundsRect = "{{0, 0}, {894, 6487}}"; @@ -4056,17 +4296,17 @@ }; 5FCC1D041594FADE0003D0E9 /* sharder_basic.cpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1153, 1105}}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 1105}}"; sepNavSelRange = "{2323, 0}"; - sepNavVisRange = "{74, 2804}"; + sepNavVisRange = "{13, 2961}"; sepNavWindowFrame = "{{61, 155}, {999, 976}}"; }; }; 5FCC1D091595000F0003D0E9 /* conversions.hpp */ = { uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {985, 8268}}"; - sepNavSelRange = "{18199, 0}"; - sepNavVisRange = "{17609, 2370}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 10374}}"; + sepNavSelRange = "{24118, 0}"; + sepNavVisRange = "{6375, 2739}"; }; }; 5FCC1D0E159501720003D0E9 /* sharder_basic */ = { @@ -4109,9 +4349,9 @@ path = ../src/preprocessing/conversions.hpp; sourceTree = ""; uiCtxt = { - sepNavIntBoundsRect = "{{0, 0}, {1268, 10179}}"; - sepNavSelRange = "{24328, 0}"; - sepNavVisRange = "{7194, 1604}"; + sepNavIntBoundsRect = "{{0, 0}, {1227, 10361}}"; + sepNavSelRange = "{8950, 0}"; + sepNavVisRange = "{5981, 3001}"; }; }; 5FCC1DD71599194B0003D0E9 /* graphchi_basic_includes.hpp */ = { @@ -4562,7 +4802,7 @@ hitCount = 0; ignoreCount = 0; lineNumber = 267; - modificationTime = 398192817.828057; + modificationTime = 398193925.409677; originalNumberOfMultipleMatches = 1; state = 0; }; diff --git a/src/api/dynamicdata/chivector.hpp b/src/api/dynamicdata/chivector.hpp index 461fea39..4b7b9190 100644 --- a/src/api/dynamicdata/chivector.hpp +++ b/src/api/dynamicdata/chivector.hpp @@ -32,6 +32,7 @@ #include #include +#include "graphchi_types.hpp" namespace graphchi { @@ -39,7 +40,7 @@ namespace graphchi { #define MINCAPACITY 2 -template +template class chivector { uint16_t nsize; diff --git a/src/preprocessing/conversions.hpp b/src/preprocessing/conversions.hpp index b643d053..af53cbc6 100644 --- a/src/preprocessing/conversions.hpp +++ b/src/preprocessing/conversions.hpp @@ -204,6 +204,7 @@ namespace graphchi { * Converts graph from an edge list format. Input may contain * value for the edges. Self-edges are ignored. */ + template void convert_edgelist(std::string inputfile, sharder &sharderobj, bool multivalue_edges=false) { diff --git a/src/preprocessing/dynamicdata/sharder.hpp b/src/preprocessing/dynamicdata/sharder.hpp index f10e6d33..6d19f926 100644 --- a/src/preprocessing/dynamicdata/sharder.hpp +++ b/src/preprocessing/dynamicdata/sharder.hpp @@ -85,7 +85,7 @@ namespace graphchi { edge_with_value(vid_t src, vid_t dst, std::vector value) : src(src), dst(dst), value(value) { } - edge_with_value(vid_t src, vid_t dst, std::vector value, HeaderDataType hdr) : src(src), dst(dst), value(value), hdr(hdr) { + edge_with_value(vid_t src, vid_t dst, std::vector value, HeaderDataType hdr) : src(src), dst(dst), hdr(hdr), value(value){ } // Order primarily by dst, then by src @@ -95,28 +95,31 @@ namespace graphchi { // TODO: use buffered I/O void reade(int f) { - read(f, &src, sizeof(vid_t)); - read(f, &dst, sizeof(vid_t)); - read(f, &is_chivec_value, sizeof(bool)); + ssize_t l; + l = read(f, &src, sizeof(vid_t)); assert(l == sizeof(vid_t)); + l = read(f, &dst, sizeof(vid_t)); assert(l == sizeof(vid_t)); + l = read(f, &is_chivec_value, sizeof(bool)); assert(l == sizeof(bool)); if (is_chivec_value) { uint16_t nvalues; - read(f, &nvalues, sizeof(uint16_t)); + l = read(f, &nvalues, sizeof(uint16_t)); assert(l == sizeof(uint16_t)); value.resize(nvalues); - read(f, &value[0], sizeof(VectorElementType) * nvalues); - read(f, &hdr, sizeof(HeaderDataType)); + // TODO: basically should loop here? + l = read(f, &value[0], sizeof(VectorElementType) * nvalues); assert(l == sizeof(VectorElementType) * nvalues); + l = read(f, &hdr, sizeof(HeaderDataType)); assert(l == sizeof(HeaderDataType)); } } void writee(int f) { - writea(f, &src, sizeof(vid_t)); - writea(f, &dst, sizeof(vid_t)); - writea(f, &is_chivec_value, sizeof(bool)); + ssize_t l; + l = write(f, &src, sizeof(vid_t)); assert(l == sizeof(vid_t)); + l = write(f, &dst, sizeof(vid_t)); assert(l == sizeof(vid_t)); + l = write(f, &is_chivec_value, sizeof(bool)); assert(l == sizeof(bool)); if (is_chivec_value) { uint16_t nvalues = value.size(); assert(value.size() < 1<<16); - writea(f, &nvalues, sizeof(uint16_t)); - writea(f, &value[0], sizeof(VectorElementType) * nvalues); - writea(f, &hdr, sizeof(HeaderDataType)); + l = write(f, &nvalues, sizeof(uint16_t)); assert(l == sizeof(uint16_t)); + l = write(f, &value[0], sizeof(VectorElementType) * nvalues); assert(l == sizeof(VectorElementType) * nvalues); + l = write(f, &hdr, sizeof(HeaderDataType)); assert(l == sizeof(HeaderDataType)); } } diff --git a/src/preprocessing/sharder.hpp b/src/preprocessing/sharder.hpp index da884d5a..240f5882 100644 --- a/src/preprocessing/sharder.hpp +++ b/src/preprocessing/sharder.hpp @@ -204,7 +204,7 @@ namespace graphchi { return idx < numedges; } - edge_with_value & next() { + edge_with_value next() { if (bufidx == bufsize_edges) { load_next(); } From 1850259c69718cc754f2e4b69bbc70857d0a0738 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Sun, 6 Oct 2013 16:38:57 -0400 Subject: [PATCH 13/14] copy constructors; use new[] instead of malloc --- src/api/dynamicdata/chivector.hpp | 8 +++ src/preprocessing/dynamicdata/sharder.hpp | 78 +++++++++++++++++++---- src/util/binary_minheap.hpp | 4 +- 3 files changed, 78 insertions(+), 12 deletions(-) diff --git a/src/api/dynamicdata/chivector.hpp b/src/api/dynamicdata/chivector.hpp index 4b7b9190..705adde4 100644 --- a/src/api/dynamicdata/chivector.hpp +++ b/src/api/dynamicdata/chivector.hpp @@ -74,6 +74,14 @@ class chivector { HeaderType & header() { return hdr; } + + const HeaderType & header() const { + return hdr; + } + + double* const& get_data() const { + return data; + } void write(T * dest) { int sz = (int) this->size(); diff --git a/src/preprocessing/dynamicdata/sharder.hpp b/src/preprocessing/dynamicdata/sharder.hpp index 3c0cf418..2a8ea1f2 100644 --- a/src/preprocessing/dynamicdata/sharder.hpp +++ b/src/preprocessing/dynamicdata/sharder.hpp @@ -88,6 +88,56 @@ namespace graphchi { edge_with_value(vid_t src, vid_t dst, std::vector value, HeaderDataType hdr) : src(src), dst(dst), hdr(hdr), value(value){ } + inline void docopy(const edge_with_value &x) { + + src = x.src; + dst = x.dst; + hdr = x.hdr; + is_chivec_value = x.is_chivec_value; + + + if (&x == this) { + // This can happen! (Also fixed an issue in binary_minhep that could cause this). + assert(false); + return; + } + + // Copy values one by one + size_t n = x.value.size(); + + if (n > 0) { + value.reserve(n); + + for(size_t i=0; i &x) : value(0) { + docopy(x); + } + + edge_with_value(edge_with_value &x) : value(0) { + docopy(x); + } + + + + edge_with_value& operator= (const edge_with_value &x) { + this->value = std::vector(0); + this->docopy(x); + return *this; + } + + edge_with_value& operator= (edge_with_value &x) { + this->value = std::vector(0); + this->docopy(x); + return *this; + } + + // Order primarily by dst, then by src bool operator< (edge_with_value &x2) { return (dst < x2.dst); @@ -168,7 +218,7 @@ namespace graphchi { } close(f); - free(buffer); + delete [] buffer; } }; @@ -308,7 +358,7 @@ namespace graphchi { virtual ~sharder() { - if (curshovel_buffer == NULL) free(curshovel_buffer); + if (curshovel_buffer == NULL) delete [] curshovel_buffer; } @@ -331,7 +381,7 @@ namespace graphchi { logstream(LOG_INFO) << "Starting preprocessing, shovel size: " << shovelsize << std::endl; - curshovel_buffer = (edge_with_value *) calloc(shovelsize, sizeof(edge_with_value)); + curshovel_buffer = new edge_with_value[shovelsize]; assert(curshovel_buffer != NULL); @@ -372,7 +422,7 @@ namespace graphchi { } shovelthreads.clear(); } - curshovel_buffer = (edge_with_value *) calloc(shovelsize, sizeof(edge_with_value)); + curshovel_buffer = new edge_with_value[shovelsize]; pthread_t t; int ret = pthread_create(&t, NULL, shard_flush_run, (void*)flushinfo); shovelthreads.push_back(t); @@ -574,7 +624,7 @@ namespace graphchi { logstream(LOG_INFO) << "Starting final processing for shard: " << shard << std::endl; std::string fname = filename_shard_adj(basefilename, shard, nshards); - std::string edfname = filename_shard_edata(basefilename, shard, nshards); + std::string edfname = filename_shard_edata(basefilename, shard, nshards); std::string edblockdirname = dirname_shard_edata_block(edfname, compressed_block_size); /* Make the block directory */ @@ -585,7 +635,7 @@ namespace graphchi { logstream(LOG_DEBUG) << "Shovel size:" << shovelsize << " edges: " << numedges << std::endl; m.start_time("finish_shard.sort"); - + std::cout << "Going to sort: " << shovelbuf[0].value.size() << std::endl; quickSort(shovelbuf, (int)numedges, edge_t_src_less); m.stop_time("finish_shard.sort"); @@ -697,7 +747,7 @@ namespace graphchi { /* Flush buffers and free memory */ writea(f, buf, bufptr - buf); free(buf); - free(shovelbuf); + delete [] shovelbuf; close(f); /* Write edata size file */ @@ -737,8 +787,13 @@ namespace graphchi { shard_capacity = (size_t) (1.2 * shard_capacity); sinkbuffer = (edge_with_value*) realloc(sinkbuffer, shard_capacity * sizeof(edge_with_value)); } - + + std::cout << cur_shard_counter << ". add: " << val.src << ", " << val.dst << ", " << val.value.size() << std::endl; + assert(val.value.size() < 10000000); sinkbuffer[cur_shard_counter++] = val; + std::cout << cur_shard_counter << ". add: " << sinkbuffer[cur_shard_counter-1].src + << ", " << sinkbuffer[cur_shard_counter-1].dst << ", " << sinkbuffer[cur_shard_counter-1].value.size() << std::endl; + prevvid = val.dst; sharded_edges++; @@ -750,7 +805,8 @@ namespace graphchi { intervals.push_back(std::pair(this_interval_start, (shardnum == nshards - 1 ? max_vertex_id : prevvid))); this_interval_start = prevvid + 1; finish_shard(shardnum++, sinkbuffer, cur_shard_counter * sizeof(edge_with_value)); - sinkbuffer = (edge_with_value *) malloc(shard_capacity * sizeof(edge_with_value)); + sinkbuffer = new edge_with_value[shard_capacity]; + //(edge_with_value *) malloc(shard_capacity * sizeof(edge_with_value)); cur_shard_counter = 0; std::cout << "Allocated sinkbuffer: " << shard_capacity * sizeof(edge_with_value) << ", shard_capacity=" << shard_capacity << std::endl; @@ -773,7 +829,7 @@ namespace graphchi { logstream(LOG_INFO) << "Created " << shardnum << " shards, expected: " << nshards << std::endl; assert(shardnum <= nshards); - free(sinkbuffer); + delete [] sinkbuffer; sinkbuffer = NULL; /* Write intervals */ @@ -820,7 +876,7 @@ namespace graphchi { shard_capacity = edges_per_shard / 2 * 3; // Shard can go 50% over shardnum = 0; this_interval_start = 0; - sinkbuffer = (edge_with_value *) calloc(shard_capacity, sizeof(edge_with_value)); + sinkbuffer = new edge_with_value [shard_capacity]; logstream(LOG_INFO) << "Edges per shard: " << edges_per_shard << " nshards=" << nshards << " total: " << shoveled_edges << std::endl; cur_shard_counter = 0; diff --git a/src/util/binary_minheap.hpp b/src/util/binary_minheap.hpp index 3126254a..710759bb 100644 --- a/src/util/binary_minheap.hpp +++ b/src/util/binary_minheap.hpp @@ -82,7 +82,9 @@ class binary_minheap { T min() { return values[0]; } void extractMin() { - values[0] = values[sz - 1]; + if (sz > 1) { + values[0] = values[sz - 1]; + } decrHeapSize(); minHeapify(0); } From bca242c11a15c49008f6f738d583f1bb0d995506 Mon Sep 17 00:00:00 2001 From: Pooja Nilangekar Date: Wed, 9 Sep 2015 12:39:48 +0300 Subject: [PATCH 14/14] fixed 'clear()' function of chivectors --- src/api/dynamicdata/chivector.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api/dynamicdata/chivector.hpp b/src/api/dynamicdata/chivector.hpp index 705adde4..af28bf27 100644 --- a/src/api/dynamicdata/chivector.hpp +++ b/src/api/dynamicdata/chivector.hpp @@ -137,6 +137,8 @@ class chivector { void clear() { nsize = 0; + if(extensions != NULL) + extensions -> clear(); } // TODO: iterators