sample_reddit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import networkx as nx\n",
    "import random\n",
    "import torch\n",
    "from train import Train_Model\n",
    "#from tqdm import tqdm, trange\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "import gc\n",
    "from model import *\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pre_sample_for_reddit(walk_times,adj_sparse, train_index, batch_size, save_name, do_walk):\n",
    "    if do_walk:\n",
    "        nodes_num = len(train_index)\n",
    "        walks = torch.zeros(nodes_num, walk_times+1).long().cuda()\n",
    "        batches = create_batches_forList(train_index, batch_size, True)\n",
    "        i=0\n",
    "        degrees = torch.tensor(adj_sparse.sum(1)).view(-1).cuda()\n",
    "        #print(degrees, (degrees<0).sum())\n",
    "        candi_node = 0\n",
    "        for batch in batches:\n",
    "            walks[i*batch_size : i*batch_size + len(batch), 0] = torch.tensor(batch).cuda()\n",
    "            candi_node = sparse_mx_to_torch_sparse_tensor(adj_sparse[batch]).cuda()\n",
    "            chosen_node = torch.zeros(len(batch), adj_sparse.shape[0]).cuda()\n",
    "            for id in range(len(batch)):\n",
    "                chosen_node[id][batch[id]] = 1.\n",
    "            candi_node = ((- chosen_node + candi_node)> 0.0).float()\n",
    "            for walk_id in range(walk_times):\n",
    "                for x in range(candi_node.shape[0]):\n",
    "                    if candi_node[x].sum()==0:\n",
    "                        #print('error')\n",
    "                        candi_node[x][batch[x]] = 1.\n",
    "                #print(candi_node.shape,candi_node)\n",
    "                #print(degrees.shape,degrees)\n",
    "                #print(candi_node.sum(1), (candi_node.sum(1)<=0).sum())\n",
    "               \n",
    "                p = candi_node * degrees\n",
    "                #print(candi_node.shape, candi_node,(candi_node.sum(1)<=0).sum())\n",
    "                #print(degrees.shape, degrees,(degrees<0).sum())\n",
    "                #print(p, (p==0).sum())\n",
    "                #print(p.shape, p,(p.sum(1)<=0).sum())\n",
    "                #p = p / (p.sum(1).unsqueeze(1))\n",
    "                #print(p, (p==0).sum())\n",
    "                #p == 1\n",
    "                #print(p,(p<0).sum())\n",
    "                new_node = torch.multinomial(p,1).squeeze(1)\n",
    "                #m = torch.distributions.categorical.Categorical(p)\n",
    "                #new_node = m.sample()\n",
    "                walks[i*batch_size : i*batch_size + len(batch), walk_id+1] = new_node\n",
    "                for id in range(len(batch)):\n",
    "                    chosen_node[id][new_node[id]] = 1.\n",
    "                candi_node = candi_node - chosen_node + sparse_mx_to_torch_sparse_tensor(adj_sparse[new_node.cpu()]).cuda()\n",
    "                candi_node = (candi_node> 0.0).float()\n",
    "                \n",
    "            i+=1\n",
    "        torch.cuda.empty_cache()\n",
    "        result1 = np.array(walks.cpu())\n",
    "        io.savemat(save_name+'.mat',{save_name:result1})\n",
    "        return walks\n",
    "    else:\n",
    "        walks = io.loadmat(save_name+'.mat')\n",
    "        return torch.tensor(walks[save_name]).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_index 152410\n",
      "val_index 23699\n",
      "test_index 55334\n",
      "adj torch.Size([232965, 232965])\n",
      "features torch.Size([232965, 602])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/ht/anaconda3/lib/python3.7/site-packages/scipy/sparse/_index.py:127: SparseEfficiencyWarning: Changing the sparsity structure of a csr_matrix is expensive. lil_matrix is more efficient.\n",
      "  self._set_arrayXarray(i, j, x)\n"
     ]
    }
   ],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2\"\n",
    "sparse_adj, sparse_adj_train,features, train_feature,labels, train_labels, id_train, id_valid, id_test, num_labels = load_data2('reddit',True)\n",
    "sparse_adj.setdiag(1.0) \n",
    "sparse_adj_train.setdiag(1.0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "process...tain_seed=0_walk=1\n",
      "process...valid_seed=0_walk=1\n",
      "process...test_seed=0_walk=1\n",
      "process...tain_seed=1_walk=1\n",
      "process...valid_seed=1_walk=1\n",
      "process...test_seed=1_walk=1\n",
      "process...tain_seed=2_walk=1\n",
      "process...valid_seed=2_walk=1\n",
      "process...test_seed=2_walk=1\n",
      "process...tain_seed=3_walk=1\n",
      "process...valid_seed=3_walk=1\n",
      "process...test_seed=3_walk=1\n",
      "process...tain_seed=4_walk=1\n",
      "process...valid_seed=4_walk=1\n",
      "process...test_seed=4_walk=1\n",
      "process...tain_seed=5_walk=1\n",
      "process...valid_seed=5_walk=1\n",
      "process...test_seed=5_walk=1\n",
      "process...tain_seed=6_walk=1\n",
      "process...valid_seed=6_walk=1\n",
      "process...test_seed=6_walk=1\n",
      "process...tain_seed=7_walk=1\n",
      "process...valid_seed=7_walk=1\n",
      "process...test_seed=7_walk=1\n",
      "process...tain_seed=8_walk=1\n",
      "process...valid_seed=8_walk=1\n",
      "process...test_seed=8_walk=1\n",
      "process...tain_seed=9_walk=1\n",
      "process...valid_seed=9_walk=1\n",
      "process...test_seed=9_walk=1\n",
      "process...tain_seed=0_walk=2\n",
      "process...valid_seed=0_walk=2\n",
      "process...test_seed=0_walk=2\n",
      "process...tain_seed=1_walk=2\n",
      "process...valid_seed=1_walk=2\n",
      "process...test_seed=1_walk=2\n",
      "process...tain_seed=2_walk=2\n",
      "process...valid_seed=2_walk=2\n",
      "process...test_seed=2_walk=2\n",
      "process...tain_seed=3_walk=2\n",
      "process...valid_seed=3_walk=2\n",
      "process...test_seed=3_walk=2\n",
      "process...tain_seed=4_walk=2\n",
      "process...valid_seed=4_walk=2\n",
      "process...test_seed=4_walk=2\n",
      "process...tain_seed=5_walk=2\n",
      "process...valid_seed=5_walk=2\n",
      "process...test_seed=5_walk=2\n",
      "process...tain_seed=6_walk=2\n",
      "process...valid_seed=6_walk=2\n",
      "process...test_seed=6_walk=2\n",
      "process...tain_seed=7_walk=2\n",
      "process...valid_seed=7_walk=2\n",
      "process...test_seed=7_walk=2\n",
      "process...tain_seed=8_walk=2\n",
      "process...valid_seed=8_walk=2\n",
      "process...test_seed=8_walk=2\n",
      "process...tain_seed=9_walk=2\n",
      "process...valid_seed=9_walk=2\n",
      "process...test_seed=9_walk=2\n",
      "process...tain_seed=0_walk=3\n",
      "process...valid_seed=0_walk=3\n",
      "process...test_seed=0_walk=3\n",
      "process...tain_seed=1_walk=3\n",
      "process...valid_seed=1_walk=3\n",
      "process...test_seed=1_walk=3\n",
      "process...tain_seed=2_walk=3\n",
      "process...valid_seed=2_walk=3\n",
      "process...test_seed=2_walk=3\n",
      "process...tain_seed=3_walk=3\n",
      "process...valid_seed=3_walk=3\n",
      "process...test_seed=3_walk=3\n",
      "process...tain_seed=4_walk=3\n",
      "process...valid_seed=4_walk=3\n",
      "process...test_seed=4_walk=3\n",
      "process...tain_seed=5_walk=3\n",
      "process...valid_seed=5_walk=3\n",
      "process...test_seed=5_walk=3\n",
      "process...tain_seed=6_walk=3\n",
      "process...valid_seed=6_walk=3\n",
      "process...test_seed=6_walk=3\n",
      "process...tain_seed=7_walk=3\n",
      "process...valid_seed=7_walk=3\n",
      "process...test_seed=7_walk=3\n",
      "process...tain_seed=8_walk=3\n",
      "process...valid_seed=8_walk=3\n",
      "process...test_seed=8_walk=3\n",
      "process...tain_seed=9_walk=3\n",
      "process...valid_seed=9_walk=3\n",
      "process...test_seed=9_walk=3\n",
      "process...tain_seed=0_walk=4\n",
      "process...valid_seed=0_walk=4\n",
      "process...test_seed=0_walk=4\n",
      "process...tain_seed=1_walk=4\n",
      "process...valid_seed=1_walk=4\n",
      "process...test_seed=1_walk=4\n",
      "process...tain_seed=2_walk=4\n",
      "process...valid_seed=2_walk=4\n",
      "process...test_seed=2_walk=4\n",
      "process...tain_seed=3_walk=4\n",
      "process...valid_seed=3_walk=4\n",
      "process...test_seed=3_walk=4\n",
      "process...tain_seed=4_walk=4\n",
      "process...valid_seed=4_walk=4\n",
      "process...test_seed=4_walk=4\n",
      "process...tain_seed=5_walk=4\n",
      "process...valid_seed=5_walk=4\n",
      "process...test_seed=5_walk=4\n",
      "process...tain_seed=6_walk=4\n",
      "process...valid_seed=6_walk=4\n",
      "process...test_seed=6_walk=4\n",
      "process...tain_seed=7_walk=4\n",
      "process...valid_seed=7_walk=4\n",
      "process...test_seed=7_walk=4\n",
      "process...tain_seed=8_walk=4\n",
      "process...valid_seed=8_walk=4\n",
      "process...test_seed=8_walk=4\n",
      "process...tain_seed=9_walk=4\n",
      "process...valid_seed=9_walk=4\n",
      "process...test_seed=9_walk=4\n",
      "process...tain_seed=0_walk=5\n",
      "process...valid_seed=0_walk=5\n",
      "process...test_seed=0_walk=5\n",
      "process...tain_seed=1_walk=5\n",
      "process...valid_seed=1_walk=5\n",
      "process...test_seed=1_walk=5\n",
      "process...tain_seed=2_walk=5\n",
      "process...valid_seed=2_walk=5\n",
      "process...test_seed=2_walk=5\n",
      "process...tain_seed=3_walk=5\n",
      "process...valid_seed=3_walk=5\n",
      "process...test_seed=3_walk=5\n",
      "process...tain_seed=4_walk=5\n",
      "process...valid_seed=4_walk=5\n",
      "process...test_seed=4_walk=5\n",
      "process...tain_seed=5_walk=5\n",
      "process...valid_seed=5_walk=5\n",
      "process...test_seed=5_walk=5\n",
      "process...tain_seed=6_walk=5\n",
      "process...valid_seed=6_walk=5\n",
      "process...test_seed=6_walk=5\n",
      "process...tain_seed=7_walk=5\n",
      "process...valid_seed=7_walk=5\n",
      "process...test_seed=7_walk=5\n",
      "process...tain_seed=8_walk=5\n",
      "process...valid_seed=8_walk=5\n",
      "process...test_seed=8_walk=5\n",
      "process...tain_seed=9_walk=5\n",
      "process...valid_seed=9_walk=5\n",
      "process...test_seed=9_walk=5\n",
      "process...tain_seed=0_walk=6\n",
      "process...valid_seed=0_walk=6\n",
      "process...test_seed=0_walk=6\n",
      "process...tain_seed=1_walk=6\n",
      "process...valid_seed=1_walk=6\n",
      "process...test_seed=1_walk=6\n",
      "process...tain_seed=2_walk=6\n",
      "process...valid_seed=2_walk=6\n",
      "process...test_seed=2_walk=6\n",
      "process...tain_seed=3_walk=6\n",
      "process...valid_seed=3_walk=6\n",
      "process...test_seed=3_walk=6\n",
      "process...tain_seed=4_walk=6\n",
      "process...valid_seed=4_walk=6\n",
      "process...test_seed=4_walk=6\n",
      "process...tain_seed=5_walk=6\n",
      "process...valid_seed=5_walk=6\n",
      "process...test_seed=5_walk=6\n",
      "process...tain_seed=6_walk=6\n",
      "process...valid_seed=6_walk=6\n",
      "process...test_seed=6_walk=6\n",
      "process...tain_seed=7_walk=6\n",
      "process...valid_seed=7_walk=6\n",
      "process...test_seed=7_walk=6\n",
      "process...tain_seed=8_walk=6\n",
      "process...valid_seed=8_walk=6\n",
      "process...test_seed=8_walk=6\n",
      "process...tain_seed=9_walk=6\n",
      "process...valid_seed=9_walk=6\n",
      "process...test_seed=9_walk=6\n",
      "process...tain_seed=0_walk=7\n",
      "process...valid_seed=0_walk=7\n",
      "process...test_seed=0_walk=7\n",
      "process...tain_seed=1_walk=7\n",
      "process...valid_seed=1_walk=7\n",
      "process...test_seed=1_walk=7\n",
      "process...tain_seed=2_walk=7\n",
      "process...valid_seed=2_walk=7\n",
      "process...test_seed=2_walk=7\n",
      "process...tain_seed=3_walk=7\n",
      "process...valid_seed=3_walk=7\n",
      "process...test_seed=3_walk=7\n",
      "process...tain_seed=4_walk=7\n",
      "process...valid_seed=4_walk=7\n",
      "process...test_seed=4_walk=7\n",
      "process...tain_seed=5_walk=7\n",
      "process...valid_seed=5_walk=7\n",
      "process...test_seed=5_walk=7\n",
      "process...tain_seed=6_walk=7\n",
      "process...valid_seed=6_walk=7\n",
      "process...test_seed=6_walk=7\n",
      "process...tain_seed=7_walk=7\n",
      "process...valid_seed=7_walk=7\n",
      "process...test_seed=7_walk=7\n",
      "process...tain_seed=8_walk=7\n",
      "process...valid_seed=8_walk=7\n",
      "process...test_seed=8_walk=7\n",
      "process...tain_seed=9_walk=7\n",
      "process...valid_seed=9_walk=7\n",
      "process...test_seed=9_walk=7\n",
      "process...tain_seed=0_walk=8\n",
      "process...valid_seed=0_walk=8\n",
      "process...test_seed=0_walk=8\n",
      "process...tain_seed=1_walk=8\n",
      "process...valid_seed=1_walk=8\n",
      "process...test_seed=1_walk=8\n",
      "process...tain_seed=2_walk=8\n",
      "process...valid_seed=2_walk=8\n",
      "process...test_seed=2_walk=8\n",
      "process...tain_seed=3_walk=8\n",
      "process...valid_seed=3_walk=8\n",
      "process...test_seed=3_walk=8\n",
      "process...tain_seed=4_walk=8\n",
      "process...valid_seed=4_walk=8\n",
      "process...test_seed=4_walk=8\n",
      "process...tain_seed=5_walk=8\n",
      "process...valid_seed=5_walk=8\n",
      "process...test_seed=5_walk=8\n",
      "process...tain_seed=6_walk=8\n",
      "process...valid_seed=6_walk=8\n",
      "process...test_seed=6_walk=8\n",
      "process...tain_seed=7_walk=8\n",
      "process...valid_seed=7_walk=8\n",
      "process...test_seed=7_walk=8\n",
      "process...tain_seed=8_walk=8\n",
      "process...valid_seed=8_walk=8\n",
      "process...test_seed=8_walk=8\n",
      "process...tain_seed=9_walk=8\n",
      "process...valid_seed=9_walk=8\n",
      "process...test_seed=9_walk=8\n",
      "process...tain_seed=0_walk=9\n",
      "process...valid_seed=0_walk=9\n",
      "process...test_seed=0_walk=9\n",
      "process...tain_seed=1_walk=9\n",
      "process...valid_seed=1_walk=9\n",
      "process...test_seed=1_walk=9\n",
      "process...tain_seed=2_walk=9\n",
      "process...valid_seed=2_walk=9\n",
      "process...test_seed=2_walk=9\n",
      "process...tain_seed=3_walk=9\n",
      "process...valid_seed=3_walk=9\n",
      "process...test_seed=3_walk=9\n",
      "process...tain_seed=4_walk=9\n",
      "process...valid_seed=4_walk=9\n",
      "process...test_seed=4_walk=9\n",
      "process...tain_seed=5_walk=9\n",
      "process...valid_seed=5_walk=9\n",
      "process...test_seed=5_walk=9\n",
      "process...tain_seed=6_walk=9\n",
      "process...valid_seed=6_walk=9\n",
      "process...test_seed=6_walk=9\n",
      "process...tain_seed=7_walk=9\n",
      "process...valid_seed=7_walk=9\n",
      "process...test_seed=7_walk=9\n",
      "process...tain_seed=8_walk=9\n",
      "process...valid_seed=8_walk=9\n",
      "process...test_seed=8_walk=9\n",
      "process...tain_seed=9_walk=9\n",
      "process...valid_seed=9_walk=9\n",
      "process...test_seed=9_walk=9\n"
     ]
    }
   ],
   "source": [
    "for walk in range(1,10):\n",
    "    for seed in range(10):\n",
    "        torch.manual_seed(seed)\n",
    "        random.seed(seed)\n",
    "        torch.cuda.manual_seed(seed)\n",
    "        np.random.seed(seed)\n",
    "        save_name = 'seed='+str(seed)+'_walk='+str(walk)\n",
    "        print('process...'+'tain_'+save_name)\n",
    "        pre_sample_for_reddit(walk,sparse_adj_train, id_train, 256, 'tain_'+save_name, True)\n",
    "        print('process...'+'valid_'+save_name)\n",
    "        pre_sample_for_reddit(walk,sparse_adj, id_valid, 256, 'valid_'+save_name, True)\n",
    "        print('process...'+'test_'+save_name)\n",
    "        pre_sample_for_reddit(walk,sparse_adj, id_test, 256, 'test_'+save_name, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}