sync_corrhdf5_files.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Copying over all forgotten files on the corr nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "from subprocess import check_output\n",
    "from dsautils.dsa_store import DsaStore\n",
    "from dsautils import cnf\n",
    "ETCD = DsaStore()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "myconf = cnf.Conf()\n",
    "params = myconf.get('corr')\n",
    "corrnames = list(params['ch0'].keys())\n",
    "filenames = [None]*len(corrnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, corr in enumerate(corrnames):\n",
    "    try:\n",
    "        filelist = check_output([\n",
    "            \"rsync\",\n",
    "            \"--list-only\",\n",
    "            \"{0}.sas.pvt:~/data/*.hdf5\".format(corr),\n",
    "            \"--exclude\",\n",
    "            \"*incomplete.hdf5\"\n",
    "        ]).decode(\"utf-8\")\n",
    "        filelist = filelist.split('\\n')\n",
    "        while '' in filelist:\n",
    "            filelist.remove('')\n",
    "        filenames[i] = sorted(file.split(' ')[-1] for file in filelist)\n",
    "    except:\n",
    "        filenames[i] = []\n",
    "for fn in filenames:\n",
    "    fn += ['9999']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "187"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum([len(fn) for fn in filenames])-len(filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['2021-08-23T07:46:49.hdf5',\n",
       "  '2021-08-23T07:51:52.hdf5',\n",
       "  '2021-08-26T09:43:10.hdf5',\n",
       "  '2021-08-27T07:54:57.hdf5',\n",
       "  '2021-08-31T08:59:07.hdf5',\n",
       "  '2021-08-31T09:09:13.hdf5',\n",
       "  '2021-09-11T14:41:16.hdf5',\n",
       "  '2021-09-24T17:12:28.hdf5',\n",
       "  '2021-09-29T12:09:45.hdf5',\n",
       "  '2021-10-01T14:56:37.hdf5',\n",
       "  '2021-10-01T15:01:40.hdf5',\n",
       "  '2021-10-03T19:12:50.hdf5',\n",
       "  '2021-10-03T19:17:52.hdf5',\n",
       "  '2021-10-03T19:22:55.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:46:47.hdf5',\n",
       "  '2021-08-23T07:51:50.hdf5',\n",
       "  '2021-08-26T09:43:09.hdf5',\n",
       "  '2021-08-27T07:54:56.hdf5',\n",
       "  '2021-09-11T14:41:15.hdf5',\n",
       "  '2021-09-24T17:12:28.hdf5',\n",
       "  '2021-09-29T12:09:43.hdf5',\n",
       "  '2021-10-03T19:12:48.hdf5',\n",
       "  '2021-10-03T19:17:51.hdf5',\n",
       "  '2021-10-03T19:22:53.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:46:47.hdf5',\n",
       "  '2021-08-23T07:51:49.hdf5',\n",
       "  '2021-08-27T07:54:56.hdf5',\n",
       "  '2021-09-11T14:41:14.hdf5',\n",
       "  '2021-09-24T17:12:26.hdf5',\n",
       "  '2021-09-29T12:09:43.hdf5',\n",
       "  '2021-10-03T19:12:48.hdf5',\n",
       "  '2021-10-03T19:17:51.hdf5',\n",
       "  '2021-10-03T19:22:53.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:51:48.hdf5',\n",
       "  '2021-08-27T07:54:56.hdf5',\n",
       "  '2021-09-11T14:41:15.hdf5',\n",
       "  '2021-10-01T14:56:37.hdf5',\n",
       "  '2021-10-03T19:22:53.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:46:46.hdf5',\n",
       "  '2021-08-23T07:51:49.hdf5',\n",
       "  '2021-08-26T09:43:09.hdf5',\n",
       "  '2021-08-27T07:54:55.hdf5',\n",
       "  '2021-09-11T14:41:14.hdf5',\n",
       "  '2021-09-22T01:25:09.hdf5',\n",
       "  '2021-09-24T17:12:27.hdf5',\n",
       "  '2021-09-29T12:09:44.hdf5',\n",
       "  '2021-10-03T19:12:49.hdf5',\n",
       "  '2021-10-03T19:17:52.hdf5',\n",
       "  '2021-10-03T19:22:55.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:46:47.hdf5',\n",
       "  '2021-08-23T07:51:49.hdf5',\n",
       "  '2021-08-27T07:54:57.hdf5',\n",
       "  '2021-09-10T02:41:10.hdf5',\n",
       "  '2021-09-11T14:41:15.hdf5',\n",
       "  '2021-09-24T17:12:27.hdf5',\n",
       "  '2021-09-29T12:09:43.hdf5',\n",
       "  '2021-10-01T14:56:37.hdf5',\n",
       "  '2021-10-03T19:12:48.hdf5',\n",
       "  '2021-10-03T19:17:51.hdf5',\n",
       "  '2021-10-03T19:22:54.hdf5',\n",
       "  '2021-10-04T21:25:12.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:46:47.hdf5',\n",
       "  '2021-08-23T07:51:49.hdf5',\n",
       "  '2021-08-26T09:43:11.hdf5',\n",
       "  '2021-08-27T07:54:57.hdf5',\n",
       "  '2021-08-31T08:59:08.hdf5',\n",
       "  '2021-08-31T09:09:13.hdf5',\n",
       "  '2021-09-10T00:19:44.hdf5',\n",
       "  '2021-09-10T02:41:03.hdf5',\n",
       "  '2021-09-11T14:41:16.hdf5',\n",
       "  '2021-09-24T17:12:30.hdf5',\n",
       "  '2021-09-29T12:09:45.hdf5',\n",
       "  '2021-10-01T14:56:52.hdf5',\n",
       "  '2021-10-01T15:01:56.hdf5',\n",
       "  '2021-10-03T19:12:51.hdf5',\n",
       "  '2021-10-03T19:17:54.hdf5',\n",
       "  '2021-10-03T19:22:57.hdf5',\n",
       "  '2021-10-04T21:25:12.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:51:47.hdf5',\n",
       "  '2021-08-27T07:54:56.hdf5',\n",
       "  '2021-09-10T00:19:44.hdf5',\n",
       "  '2021-09-11T14:41:16.hdf5',\n",
       "  '2021-09-24T17:12:27.hdf5',\n",
       "  '2021-09-29T12:09:44.hdf5',\n",
       "  '2021-10-01T14:56:37.hdf5',\n",
       "  '2021-10-01T15:01:40.hdf5',\n",
       "  '2021-10-03T19:12:48.hdf5',\n",
       "  '2021-10-03T19:17:51.hdf5',\n",
       "  '2021-10-03T19:22:54.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:46:45.hdf5',\n",
       "  '2021-08-23T07:51:48.hdf5',\n",
       "  '2021-08-27T07:54:56.hdf5',\n",
       "  '2021-09-11T14:41:14.hdf5',\n",
       "  '2021-10-03T19:17:51.hdf5',\n",
       "  '2021-10-03T19:22:53.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:46:47.hdf5',\n",
       "  '2021-08-23T07:51:50.hdf5',\n",
       "  '2021-08-26T09:43:11.hdf5',\n",
       "  '2021-08-27T07:54:59.hdf5',\n",
       "  '2021-08-31T08:59:09.hdf5',\n",
       "  '2021-08-31T09:09:15.hdf5',\n",
       "  '2021-09-10T00:19:47.hdf5',\n",
       "  '2021-09-11T14:41:17.hdf5',\n",
       "  '2021-09-24T17:12:30.hdf5',\n",
       "  '2021-09-29T12:09:45.hdf5',\n",
       "  '2021-10-01T14:56:40.hdf5',\n",
       "  '2021-10-01T15:01:43.hdf5',\n",
       "  '2021-10-03T19:12:51.hdf5',\n",
       "  '2021-10-03T19:17:54.hdf5',\n",
       "  '2021-10-03T19:22:57.hdf5',\n",
       "  '2021-10-04T21:25:13.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:46:48.hdf5',\n",
       "  '2021-08-23T07:51:51.hdf5',\n",
       "  '2021-08-24T08:24:05.hdf5',\n",
       "  '2021-08-27T07:54:56.hdf5',\n",
       "  '2021-09-10T00:19:46.hdf5',\n",
       "  '2021-09-10T02:41:04.hdf5',\n",
       "  '2021-09-11T14:41:17.hdf5',\n",
       "  '2021-09-24T17:12:28.hdf5',\n",
       "  '2021-09-29T12:09:43.hdf5',\n",
       "  '2021-10-01T14:56:38.hdf5',\n",
       "  '2021-10-01T15:01:41.hdf5',\n",
       "  '2021-10-03T19:12:51.hdf5',\n",
       "  '2021-10-03T19:17:54.hdf5',\n",
       "  '2021-10-03T19:22:57.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:46:46.hdf5',\n",
       "  '2021-08-23T07:51:49.hdf5',\n",
       "  '2021-08-27T07:54:57.hdf5',\n",
       "  '2021-09-11T14:41:14.hdf5',\n",
       "  '2021-09-29T12:09:44.hdf5',\n",
       "  '2021-10-03T19:12:49.hdf5',\n",
       "  '2021-10-03T19:17:51.hdf5',\n",
       "  '2021-10-03T19:22:54.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:46:48.hdf5',\n",
       "  '2021-08-23T07:51:50.hdf5',\n",
       "  '2021-08-26T09:43:09.hdf5',\n",
       "  '2021-08-27T07:54:57.hdf5',\n",
       "  '2021-08-31T08:59:07.hdf5',\n",
       "  '2021-08-31T09:09:13.hdf5',\n",
       "  '2021-09-10T00:19:44.hdf5',\n",
       "  '2021-09-11T14:41:15.hdf5',\n",
       "  '2021-09-29T12:09:44.hdf5',\n",
       "  '2021-10-01T14:56:37.hdf5',\n",
       "  '2021-10-01T15:01:40.hdf5',\n",
       "  '2021-10-03T19:12:48.hdf5',\n",
       "  '2021-10-03T19:17:51.hdf5',\n",
       "  '2021-10-03T19:22:53.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:46:47.hdf5',\n",
       "  '2021-08-23T07:51:50.hdf5',\n",
       "  '2021-08-27T07:54:56.hdf5',\n",
       "  '2021-09-10T00:19:44.hdf5',\n",
       "  '2021-09-10T02:41:07.hdf5',\n",
       "  '2021-09-11T14:41:15.hdf5',\n",
       "  '2021-09-24T17:12:27.hdf5',\n",
       "  '2021-09-29T12:09:44.hdf5',\n",
       "  '2021-10-01T14:56:37.hdf5',\n",
       "  '2021-10-03T19:12:48.hdf5',\n",
       "  '2021-10-03T19:17:51.hdf5',\n",
       "  '2021-10-03T19:22:54.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:46:46.hdf5',\n",
       "  '2021-08-23T07:51:49.hdf5',\n",
       "  '2021-08-26T09:43:09.hdf5',\n",
       "  '2021-08-27T07:54:57.hdf5',\n",
       "  '2021-08-31T08:59:08.hdf5',\n",
       "  '2021-08-31T09:09:13.hdf5',\n",
       "  '2021-09-10T00:19:43.hdf5',\n",
       "  '2021-09-11T14:41:15.hdf5',\n",
       "  '2021-09-24T17:12:28.hdf5',\n",
       "  '2021-10-01T14:56:37.hdf5',\n",
       "  '2021-10-01T15:01:40.hdf5',\n",
       "  '2021-10-03T19:12:50.hdf5',\n",
       "  '2021-10-03T19:17:52.hdf5',\n",
       "  '2021-10-03T19:22:55.hdf5',\n",
       "  '9999'],\n",
       " ['2021-08-23T07:46:46.hdf5',\n",
       "  '2021-08-23T07:51:49.hdf5',\n",
       "  '2021-08-27T07:54:57.hdf5',\n",
       "  '2021-08-31T09:09:12.hdf5',\n",
       "  '2021-09-10T02:41:01.hdf5',\n",
       "  '2021-09-11T14:41:15.hdf5',\n",
       "  '2021-09-24T17:12:28.hdf5',\n",
       "  '2021-09-29T12:09:43.hdf5',\n",
       "  '2021-10-01T14:56:39.hdf5',\n",
       "  '2021-10-01T15:01:42.hdf5',\n",
       "  '2021-10-03T19:12:49.hdf5',\n",
       "  '2021-10-03T19:17:52.hdf5',\n",
       "  '2021-10-03T19:22:55.hdf5',\n",
       "  '2021-10-04T21:25:11.hdf5',\n",
       "  '9999']]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filenames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-110dac9fd059>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     13\u001b[0m         )\n\u001b[1;32m     14\u001b[0m     \u001b[0mfilenames\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcorridx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremove\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m     \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "while sum([len(fn) for fn in filenames]) > len(filenames)*2-1:\n",
    "    nextfiles = [fn[0] for fn in filenames]\n",
    "    file = min(nextfiles)\n",
    "    corridx = nextfiles.index(file)\n",
    "    if '2021-10' in file:\n",
    "        ETCD.put_dict(\n",
    "            '/cmd/cal',\n",
    "            {'cmd': 'rsync',\n",
    "             'val': {'hostname': corrnames[corridx],\n",
    "                     'filename': '/home/ubuntu/data/{0}'.format(file)\n",
    "            }\n",
    "            }\n",
    "        )\n",
    "    filenames[corridx].remove(file)\n",
    "    time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}